增加邮件推送功能

README.md

@@ -3,6 +3,13 @@
 2019/9/12
 密钥长度由1024修改为4096
 
+dev2.3.2.3 2019/11/14
+1、增加定时邮件推送功能，默认推送昨天的数据，推送过的不再推
+使用item区分同用户下不同的收件人
+2、对字段的定义，按顺序从config.json中读取fields字段，生成excel动态根据用户配置的字段生成
+3、增加手动发送邮件的接口
+4、增加api_test
+
 de2.3.2.2  2019/9/6
 1、增加每个用户的自定义字段，是加或是减
 fields:"subtype,-toptype"

api_test/main.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"strings"
+	"time"
+)
+
+var (
+	appid  = "jyNjdXQgUDAwdaTklMPz5i"
+	key    = "404M0v2j"
+	apiurl = "http://127.0.0.1:8801"
+
+	//apiurl = "https://api.jianyu360.com"
+)
+
+func main() {
+	getData()
+}
+
+func getToken() (token string) {
+	tm := fmt.Sprintf("%d", time.Now().Unix())
+	res := post(apiurl+"/user/access_token", map[string]string{
+		"appid":     appid,
+		"timestamp": tm,
+		"signature": MD5(appid + tm + key),
+	})
+	if res != nil && res["access_token"] != "" {
+		token, _ = res["access_token"].(string)
+	}
+	return
+}
+
+func getData() {
+	token := getToken()
+	data := post(apiurl+"/data/getalldata", map[string]string{
+		"access_token": token,
+		"day":          "-1",
+	})
+	log.Println(data)
+}
+
+func MD5(str string) string {
+	h := md5.New()
+	h.Write([]byte(str))
+	return strings.ToUpper(hex.EncodeToString(h.Sum(nil)))
+}
+
+func post(url string, form map[string]string) (data map[string]interface{}) {
+	str := ""
+	for k, v := range form {
+		str += "&" + k + "=" + v
+	}
+	res, err := http.Post(url, "application/x-www-form-urlencoded", strings.NewReader(str))
+	if err != nil {
+		log.Println("post err:", err.Error())
+	} else if res.Body != nil {
+		defer res.Body.Close()
+		bs, _ := ioutil.ReadAll(res.Body)
+		json.Unmarshal(bs, &data)
+	}
+	return
+}

jyservice/src/config.json

@@ -1,25 +1,20 @@
 {
     "webport": ":8801",
-    "jkmail": {
-        "to": "renzheng@topnet.net.cn",
-        "api": "http://10.171.112.160:19281/_send/_mail",
-        "cron-alert": "0 30 8 * * ?",
-        "cron-check": "0 0 17 * * ?",
-		"cron-tj": "0 12 0,16 * * ?"
-    },
     "mongodb": {
         "addr": "192.168.3.207:27082",
         "db": "jyqyfw",
-        "pool": 10
+        "pool": 4
+    },
+    "jkmail": {
+        "to": "renzheng@topnet.net.cn,bd@topnet.net.cn",
+        "api": "http://10.171.112.160:19281/_send/_mail",
+        "cron-alert": "0 22 11,17 * * ?",
+        "cron-tj": "0 23 0,13 * * ?"
     },
     "redis": {
-        "addr": "jyqyfw=192.168.3.207:1380",
-        "pool": 50
+        "addr": "jyqyfw=192.168.3.207:1379",
+        "pool": 4
     },
-    "cassandra": [
-        "192.168.3.18"
-    ],
-    "cassandrasize": 5,
     "loglevel": 5,
     "plan": {
         "A": {
@@ -28,8 +23,8 @@
             "publishtime": 1,
             "href": 1,
             "id": 1,
-            "department": 1,
             "matchscore": 1,
+            "department": 1,
             "remark": 1
         },
         "B": {
@@ -47,13 +42,54 @@
             "buyerperson": 1,
             "city": 1,
             "id": 1,
-            "department": 1,
             "matchscore": 1,
+            "department": 1,
             "remark": 1,
             "priority": 1,
             "industry": 1,
             "matchtype": 1,
-            "matchkey": 1
+            "matchkey": 1,
+            "s_winner": 1
         }
+    },
+    "mail": [
+        {
+            "addr": "smtp.exmail.qq.com",
+            "port": 465,
+            "pwd": "ue9Rg9Sf4CVtdm5a",
+            "user": "public03@topnet.net.cn"
+        }
+    ],
+    "cron-sendmail": "0 20 16 * * ?",
+    "fields": {
+        "title": "公告标题",
+        "content": "公告内容",
+        "href": "公告地址",
+        "jybxhref": "剑鱼标讯地址",
+		"detail":"公告正文",
+        "area": "省份",
+        "city": "城市",
+        "publishtime": "发布时间",
+        "projectname": "项目名称",
+        "projectcode": "项目编号",
+        "buyer": "采购单位",
+        "buyerperson": "采购单位联系人",
+        "buyertel": "采购单位联系电话",
+        "budget": "预算(元)",
+        "bidopentime": "开标日期",
+        "s_winner": "中标单位",
+        "bidamount": "中标价(元)",
+        "agency": "招标机构",
+        "projectscope": "项目范围",
+        "currency": "币种",
+        "subtype": "公告类别",
+        "toptype": "公告类别大类",
+        "id": "信息标识",
+        "industry": "行业",
+        "matchkey": "信息匹配词",
+        "matchtype": "匹配字段",
+        "priority": "信息重要程度",
+        "department": "部门",
+        "excludetype": "排除类型"
     }
 }

jyservice/src/main.go

@@ -1,5 +1,6 @@
 package main
 
+//-----linux
 import (
 	"usermanager"
 	"utils"
@@ -12,6 +13,7 @@ func Server() {
 	endless.ListenAndServe(utils.Sysconfig["webport"].(string), usermanager.Middleware(usermanager.Route))
 }
 
+//-----windows
 //import (
 //	"net/http"
 //	"usermanager"

jyservice/src/usermanager/sendmail.go

@@ -0,0 +1,352 @@
+package usermanager
+
+import (
+	"net/http"
+	"os"
+	qu "qfw/util"
+	"qfw/util/redis"
+	"strings"
+	"sync"
+	"time"
+	. "utils"
+
+	"qfw/util/mail"
+
+	"github.com/robfig/cron"
+	log "github.com/sirupsen/logrus"
+	"github.com/tealeg/xlsx"
+)
+
+//给用户发送邮件
+var Gmails []*mail.GmailAuth
+
+func init() {
+	mails, _ := Sysconfig["mail"].([]interface{})
+	if len(mails) > 0 {
+		Gmails = make([]*mail.GmailAuth, len(mails))
+		for k, v1 := range mails {
+			v, _ := v1.(map[string]interface{})
+			Gmails[k] = &mail.GmailAuth{
+				SmtpHost: qu.ObjToString(v["addr"]),
+				SmtpPort: qu.IntAll(v["port"]),
+				User:     qu.ObjToString(v["user"]),
+				Pwd:      qu.ObjToString(v["pwd"]),
+				//PoolSize: v.MailPollSize,
+				//ReTry:    v.MailReTry,
+			}
+		}
+	}
+	go func() {
+		time.Sleep(5 * time.Second)
+		cron1, _ := Sysconfig["cron-sendmail"].(string)
+		if len(cron1) > 2 {
+			log.Debug("enter sendMailtask...")
+			c := cron.New()
+			log.Println("发送数据定时任务")
+			c.AddFunc(cron1, sm)
+			c.Start()
+			defer c.Stop()
+			select {}
+		}
+	}()
+}
+
+func sm() {
+	sendmail("", -1, 1, "")
+}
+
+//调用url地址发送邮件
+func SendMailByAppid(w http.ResponseWriter, r *http.Request) {
+	defer qu.Catch()
+	appid := r.FormValue("appid") //用户身份
+	days := r.FormValue("days")   //往前取几天的数据
+	force := r.FormValue("force") //强制获取忽略bget状态
+	fn := r.FormValue("fn")       //文件名
+	if len(appid) > 0 {
+		day := -1
+		if days != "" {
+			day = qu.IntAll(days)
+		}
+		force1 := qu.IntAll(force)
+		sendmail(appid, day, force1, fn)
+		w.Write([]byte("ok"))
+	} else {
+		w.Write([]byte("param error"))
+	}
+}
+
+//定时发送邮件
+/**
+appid 用户身份
+days -1发送昨天数据 -3发送前三天数据
+force 1不判断bget 0判断bget为1的不发送
+**/
+func sendmail(appid string, days int, force int, fn string) {
+	query := map[string]interface{}{
+		"mails": map[string]interface{}{
+			"$exists": 1,
+		},
+	}
+	if len(appid) > 6 {
+		query = map[string]interface{}{
+			"appid": appid,
+		}
+	}
+	res, b := Mgo.Find("user", query, nil, nil, false, -1, -1)
+	if b && res != nil && *res != nil && len(*res) > 0 {
+		for _, v := range *res {
+			if !(v["bnormal"] == nil || qu.IntAll(v["bnormal"]) == 1) {
+				log.Println("用户为关闭状态", appid)
+				continue
+			}
+			appid, _ := v["appid"].(string)
+			//h := qu.IntAll(v["sendmailhour"])
+			//			key := fmt.Sprintf("sendmail_%d_%s", tn.Day(), appid)
+			//			b, _ := redis.Exists(REDISDB, key)
+			plan, _ := v["plan"].(map[string]interface{})
+			planname := ""
+			if len(plan) > 0 && plan["name"] != nil {
+				planname, _ = plan["name"].(string)
+			}
+			appfields, _ := v["fields"].(string)
+			mails, _ := v["mails"].(map[string]interface{})
+			if len(mails) > 0 && appid != "" && len(planname) > 1 {
+				GetDataMapLock.Lock()
+				appidLock := GetDataMap[appid]
+				if appidLock == nil {
+					appidLock = &sync.Mutex{}
+					GetDataMap[appid] = appidLock
+				}
+				GetDataMapLock.Unlock()
+				appidLock.Lock()
+				defer appidLock.Unlock()
+				log.Println(appid, mails)
+
+				//判断剩余服务条数
+				limitnum := redis.Get(REDISDB, "limitnum_"+appid) //值
+				if limitnum == nil {                              //值为空时从数据库中提取
+					limitnum = plan["current"]
+					if limitnum != nil {
+						redis.Put(REDISDB, "limitnum_"+appid, limitnum, 0)
+					}
+				}
+				num := qu.IntAll(limitnum)
+				if num < 1 {
+					log.Println("用户额度已使用完", appid)
+					continue
+				}
+				//最近7天，bget不等于1套餐
+				dataQuery := map[string]interface{}{
+					"appid": appid,
+				}
+				st, _ := GetDayMinMax(time.Now())
+				st1 := st + int64(days*86400)
+				dataQuery["createtime"] = map[string]interface{}{
+					"$gte": st1,
+					"$lt":  st,
+				}
+				if fn == "" {
+					fn = qu.FormatDateByInt64(&st1, qu.Date_Short_Layout) + "_" + qu.FormatDateByInt64(&st, qu.Date_Short_Layout)
+				}
+				if force == 0 {
+					dataQuery["bget"] = map[string]interface{}{
+						"$ne": 1,
+					}
+				}
+				pn := planname[:1]
+				var fields map[string]interface{}
+				if pn == "A" {
+					fields = A
+				} else if pn == "B" {
+					fields = B
+				} else if pn == "C" {
+					fields = C
+				} else if pn == "D" {
+					fields = D
+				} else if pn == "E" {
+					fields = E
+				} else if pn == "F" {
+					fields = F
+				}
+				fields2 := map[string]interface{}{}
+				for k1, v1 := range fields {
+					fields2[k1] = v1
+				}
+				//增加自定义字段控制 2019/9/6
+				if appfields != "" {
+					FS := strings.Split(appfields, ",")
+					for _, vf := range FS {
+						if strings.HasPrefix(vf, "-") && len(vf) > 1 {
+							delete(fields2, vf[1:])
+						} else {
+							fields2[vf] = 1
+						}
+					}
+				}
+				fields2["item"] = 1
+				log.Info(fields2)
+				data, bdata := Mgo.Find("usermail", dataQuery, `{"_id":1}`, fields2, false, -1, -1)
+				firstId, lastId := "", ""
+				if bdata && data != nil && *data != nil && len(*data) > 0 {
+					infos := *data
+					allUserData := map[string][]map[string]interface{}{}
+					updateMap := [][]map[string]interface{}{}
+					newGetLen := 0
+					for index, v1 := range infos {
+						if qu.IntAll(v1["bget"]) == 0 {
+							newGetLen++
+							updateMap = append(updateMap, []map[string]interface{}{
+								map[string]interface{}{
+									"_id": v1["_id"],
+								},
+								map[string]interface{}{
+									"$set": map[string]interface{}{
+										"bget": 1,
+									},
+								},
+							})
+						}
+						if index == 0 {
+							firstId = qu.BsonIdToSId(v1["_id"])
+						}
+						if index == len(infos)-1 {
+							lastId = qu.BsonIdToSId(v1["_id"])
+						}
+						id, _ := v1["id"].(string)
+						v1["id"] = SE.EncodeString(id)
+						delete(v1, "_id")
+						delete(v1, "bget")
+						if qu.Int64All(v1["bidopentime"]) == 0 {
+							delete(v1, "bidopentime")
+						} else {
+							bo := v1["bidopentime"]
+							v1["bidopentime"] = qu.FormatDateWithObj(&bo, qu.Date_Full_Layout)
+						}
+						if qu.Int64All(v1["publishtime"]) > 0 {
+							bo := v1["publishtime"]
+							v1["publishtime"] = qu.FormatDateWithObj(&bo, qu.Date_Full_Layout)
+						}
+						item, _ := v1["item"].(string)
+						if item == "" {
+							log.Info("info err,item nil")
+							continue
+						}
+						userMap := allUserData[item]
+						if userMap == nil {
+							userMap = []map[string]interface{}{}
+						}
+						userMap = append(userMap, v1)
+						allUserData[item] = userMap
+					}
+					//处理总条数,有重复获取的信息条不计数，*******************
+					if newGetLen > 0 {
+						redis.Decrby(REDISDB, "limitnum_"+appid, newGetLen)
+						go Mgo.UpdateBulk("usermail", updateMap...)
+					}
+					d := map[string]interface{}{}
+					d["size"] = len(infos)
+					d["new_size"] = newGetLen
+					d["available_infos"] = qu.IntAll(limitnum) - newGetLen
+					go func() {
+						saveData := map[string]interface{}{
+							"appid":   appid,
+							"flag":    "sendmail",
+							"alltype": 2,
+							"data":    d,
+							"date":    time.Now().Unix(),
+							"first":   firstId,
+							"last":    lastId,
+						}
+						if Mgo.Save("userdatalog", saveData) == "" {
+							log.Println(saveData)
+						}
+					}()
+					GetBidInfoXlsx(appid, fn, mails, fields2, allUserData)
+				}
+			}
+		}
+	}
+
+}
+
+func GetBidInfoXlsx(appid string, fn string, mails map[string]interface{}, fields map[string]interface{}, allUserData map[string][]map[string]interface{}) {
+	//信息
+	for key, data := range allUserData {
+		email, _ := mails[key].(string)
+		if email == "" {
+			log.Info("mail nil error", appid, key)
+			continue
+		}
+		fx := xlsx.NewFile()
+		sheet := xlsx.Sheet{}
+
+		style := xlsx.NewStyle()
+		border := *xlsx.NewBorder("thin", "thin", "thin", "thin")
+		style.Border = border
+		style.ApplyBorder = true
+		style.Alignment.WrapText = true
+
+		//生表头
+		styleHead := xlsx.NewStyle()
+		styleHead.Alignment.WrapText = true
+		styleHead.Font.Bold = true
+		styleHead.Border = border
+		styleHead.ApplyBorder = true
+		head := sheet.AddRow()
+		arr := []string{}
+		for _, t := range FielsArr {
+			if fields[t] != nil {
+				arr = append(arr, t)
+				cell := head.AddCell()
+				cell.SetValue(Fiels[t])
+				cell.SetStyle(styleHead)
+			}
+		}
+
+		for _, v := range data {
+			row := sheet.AddRow()
+			for _, t := range arr {
+				cell := row.AddCell()
+				if v[t] != nil {
+					cell.SetValue(v[t])
+					cell.SetStyle(style)
+				}
+			}
+		}
+
+		fx.AppendSheet(sheet, "剑鱼标讯")
+		//t := time.Now()
+		dir := "./xlsx/" + appid + "/" + key + "/"
+		if b, _ := PathExists(dir); !b {
+			err1 := os.MkdirAll(dir, os.ModePerm)
+			if err1 != nil {
+				log.Println("mkdir err", dir)
+			}
+		}
+		fname := key + "_" + fn + qu.GetRandom(4) + ".xlsx"
+		err := fx.Save(dir + fname)
+		if err != nil {
+			log.Info("xls error", appid, fname)
+		} else {
+			for i := 0; i < len(Gmails); i++ {
+				gmail := Gmails[i]
+				status := mail.GSendMail("剑鱼标讯", email, "", "", key+"_"+fn, "", dir+fname, fname, gmail)
+				if status {
+					log.Println("send mail success", appid, email)
+					break
+				}
+			}
+		}
+	}
+}
+
+func PathExists(path string) (bool, error) {
+	_, err := os.Stat(path)
+	if err == nil {
+		return true, nil
+	}
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+	return false, err
+}

jyservice/src/usermanager/usermanager.go

@@ -28,6 +28,8 @@ func init() {
 	//管理地址
 	Route.HandleFunc("/api1/user/jy_newuser", NewUser).Methods("POST")
 	Route.HandleFunc("/api1/user/jy_newservice", NewService).Methods("POST")
+	//手动发送邮件
+	Route.HandleFunc("/api1/data/sendmail", SendMailByAppid).Methods("GET", "POST")
 
 	//http.Handle("/", middleware(Route))
 	go SaveLogTask()

jyservice/src/utils/utils.go

@@ -1,9 +1,12 @@
 package utils
 
 import (
+	"io/ioutil"
 	"os"
 	"qfw/util"
 	"qfw/util/redis"
+	"regexp"
+	"strings"
 	"time"
 
 	log "github.com/sirupsen/logrus"
@@ -45,11 +48,32 @@ const (
 var (
 	Sysconfig        map[string]interface{}
 	A, B, C, D, E, F map[string]interface{}
+	Fiels            = map[string]string{}
+	FielsArr         = []string{}
 )
 
 func init() {
 	//初始化日志
 	util.ReadConfig(&Sysconfig)
+	r, _ := os.Open("./config.json")
+	bs, _ := ioutil.ReadAll(r)
+	r.Close()
+	jsonStr := string(bs)
+	index := strings.Index(jsonStr, `"fields":`)
+	if index > 0 {
+		jsonStr = jsonStr[index+9:]
+		index = strings.Index(jsonStr, `}`)
+		if index > 0 {
+			jsonStr = jsonStr[:index]
+		}
+	}
+	fs := regexp.MustCompile(`"([a-zA-Z0-9_]+)"[:]`).FindAllStringSubmatch(jsonStr, -1)
+	for _, v1 := range fs {
+		if len(v1) == 2 {
+			FielsArr = append(FielsArr, v1[1])
+		}
+	}
+	log.Println(FielsArr)
 	t := &log.TextFormatter{}
 	t.TimestampFormat = "01-02 15:04:05.000"
 	t.ForceColors = true
@@ -59,6 +83,13 @@ func init() {
 	log.SetOutput(os.Stdout)
 	log.SetLevel(log.Level(util.IntAllDef(Sysconfig["loglevel"], 5)))
 	//
+	f1 := Sysconfig["fields"]
+	if f1 != nil {
+		fm, _ := f1.(map[string]interface{})
+		for k, v := range fm {
+			Fiels[k], _ = v.(string)
+		}
+	}
 	plan, _ := Sysconfig["plan"].(map[string]interface{})
 	A, _ = plan["A"].(map[string]interface{})
 	B, _ = plan["B"].(map[string]interface{})