dataprocess_file/main.go

package main

import (
	"encoding/json"
	"fmt"
	"github.com/nats-io/nats.go"
	cu "jygit.jydev.jianyu360.cn/data_capture/myself_util/commonutil"
	iu "jygit.jydev.jianyu360.cn/data_capture/myself_util/initutil"
	su "jygit.jydev.jianyu360.cn/data_capture/myself_util/spiderutil"
	"log"
	"net/http"
	"time"
)

var (
	Config    map[string]interface{}
	Webport   string
	Subscribe string
	Api       string
	To        string
)

func init() {
	iu.ReadConfig(&Config)
	InitFileInfo() //初始化附件解析信息
	InitOss()      //oss
	InitNats()     //nats
	Webport = cu.ObjToString(Config["webport"])
	Subscribe = cu.ObjToString(Config["subscribe"])
	Api = cu.ObjToString(Config["api"])
	To = cu.ObjToString(Config["to"])
}
func main() {
	go http.ListenAndServe(":"+Webport, nil)
	SubscribeNats()
	ch := make(chan bool, 1)
	<-ch
}

func SubscribeNats() {
	//先消费，带压缩
	Jnats.SubZip(Subscribe, func(msg *nats.Msg) {
		data := &MsgInfo{}
		err := json.Unmarshal(msg.Data, &data)
		if err != nil {
			log.Println("解析数据失败：", err)
			data.Err = err
			//SaveData()//保存异常数据
		} else {
			//处理数据
			data.Stime = time.Now().Unix()
			data.CurrSetp = Subscribe
			DealFile(data.Data)
			data.Etime = time.Now().Unix()
		}
		//消息回写
		bs, _ := json.Marshal(data)
		err = msg.Respond(bs)
		if err != nil {
			fmt.Println("回执失败：", data.Id)
			//SaveData()//保存异常数据
		}
	})
}

func DealFile(tmp map[string]interface{}) {
	site := cu.ObjToString(tmp["site"])              //解析附件站点
	if limitRatio := OssSite[site]; limitRatio > 0 { //配置站点解析附件，根据准确率情况替换正文
		replace, filetext := AnalysisFile(true, limitRatio, tmp)
		if replace { //替换正文
			tmp["detail"] = filetext
		}
	} else { //其它网站附件信息，detail无效，只有一个附件且不是ocr识别的，替换正文
		//判断detail是否有效
		detail := cu.ObjToString(tmp["detail"])
		detail = su.FilterDetail(detail) //只保留文本内容
		if len([]rune(detail)) <= 5 || (len([]rune(detail)) <= 50 && SpecialTextReg.MatchString(detail)) {
			replace, filetext := AnalysisFile(false, 0, tmp)
			if replace { //替换正文
				tmp["detail"] = filetext
			}
		}
	}
}

/*func SendMail(bodyTextAll string) {
	res, err := http.Get(fmt.Sprintf("%s?to=%s&title=%s&body=%s", Api, To, "ocr_file_over", bodyTextAll))
	if err == nil {
		defer res.Body.Close()
		read, err := ioutil.ReadAll(res.Body)
		fmt.Println("邮件发送成功：", string(read), err)
	} else {
		fmt.Println("邮件发送失败：", err)
	}
}
*/