邮件

api_test/config.json

@@ -1,6 +1,6 @@
 {
-    "appid": "jyPRZXQgoJCANdSUBBAiNr",
-    "key": "B00Wi4Cg",
-    "day": "-1",
+    "appid": "jyMhhXQQcIAAZZT0JKCTFX",
+    "key": "x05Sn3cH",
+    "day": "0",
     "apiurl": "https://api.jianyu360.com"
 }

api_test/main.go

@@ -11,6 +11,8 @@ import (
 	"net/http"
 	"net/url"
 	"qfw/util"
+	"qfw/util/mysql"
+	"strconv"
 	"strings"
 	"time"
 )
@@ -24,10 +26,20 @@ var (
 	IdleConnTimeout         int = 2048
 	ConnectTimeOut          int = 30
 	KeepAlive               int = 30
+	Tidb                    *mysql.Mysql
 )
 
 func init() {
 	httpClient = createHttpClient()
+	Tidb = &mysql.Mysql{
+		Address:      "192.168.3.217:4000",
+		UserName:     "root",
+		PassWord:     "=PDT49#80Z!RVv52_z",
+		DBName:       "createxzh",
+		MaxOpenConns: 20,
+		MaxIdleConns: 40,
+	}
+	Tidb.Init()
 }
 
 func main() {
@@ -73,20 +85,49 @@ func getToken() (token string) {
 
 func getData() {
 	// token := getToken()
-	tm := fmt.Sprint(time.Now().Unix())
-	data := post(apiurl+"/data/getalldata", map[string]string{
-		// "access_token": token,
-		"day":   day,
-		"next":  "0",
-		"appid": appid,
-	}, map[string]string{
-		"timestamp": tm,
-		"signature": MD5(appid + tm + key),
-	})
-	//s, _ := json.Marshal(data["data"])
-	//delete(data, "data")
-	log.Println("tm", appid, tm)
-	log.Println(MD5(appid+tm+key), data)
+	log.Println("开始任务")
+	next := "0"
+	for {
+		tm := fmt.Sprint(time.Now().Unix())
+		data := post(apiurl+"/data/getalldata", map[string]string{
+			// "access_token": token,
+			"day":   day,
+			"next":  next,
+			"appid": appid,
+		}, map[string]string{
+			"timestamp": tm,
+			"signature": MD5(appid + tm + key),
+		})
+		if data != nil {
+			log.Println(data["next"])
+			datas := util.ObjArrToMapArr(data["data"].([]interface{}))
+			for _, v := range datas {
+				filehref, _ := json.Marshal(v["filehref"])
+				v["filehref"] = string(filehref)
+				purchasinglist, _ := json.Marshal(v["purchasinglist"])
+				v["purchasinglist"] = string(purchasinglist)
+				id := util.ObjToString(v["id"])
+				if Tidb.Count("info", map[string]interface{}{"id": id}) == 0 {
+					did := Tidb.Insert("info", v)
+					if did > 0 {
+						log.Println("保存成功 ", id)
+					}
+				} else {
+					log.Println("库中已有 ", id)
+				}
+			}
+			nexts := util.IntAll(data["next"])
+			if nexts == 0 {
+				break
+			} else {
+				tm = fmt.Sprint(time.Now().Unix())
+				next = strconv.Itoa(nexts)
+				log.Println("第", next, "条")
+			}
+		} else {
+			break
+		}
+	}
 }
 
 func MD5(str string) string {
@@ -119,7 +160,9 @@ func post(urls string, form, header map[string]string) (data map[string]interfac
 		log.Println("read reply error:", err)
 		return
 	}
-	log.Println(string(replay))
+	// 创建一个空的map来存放解析后的数据
+
+	// 使用json.Unmarshal来解析[]byte到map
 	json.Unmarshal(replay, &data)
 	return
 }

jyservice/src/usermanager/getdata.go

@@ -149,6 +149,7 @@ func CheckUserInfo(access_token string, day, next, all int, signature, timestamp
 			h := md5.New()
 			h.Write([]byte(appid + timestamp + util.ObjToString((*data)["key"])))
 			s1 := strings.ToUpper(hex.EncodeToString(h.Sum(nil)))
+			log.Println("appid:", appid, timestamp, util.ObjToString((*data)["key"]))
 			log.Println("signature:", signature, s1)
 			if s1 == signature {
 				bcheck = true
@@ -971,7 +972,7 @@ func PushLtData(w http.ResponseWriter, r *http.Request) {
 	WriteJSON(w, &ret)
 }
 
-//拓普标讯信息
+// 拓普标讯信息
 func TopInfoData(w http.ResponseWriter, r *http.Request) {
 	//身份参数
 	appid := r.FormValue("appid")

jyservice/src/usermanager/sendmail.go

@@ -3,6 +3,7 @@ package usermanager
 import (
 	. "config"
 	"fmt"
+	"mongodb"
 	"net/http"
 	"os"
 	mongoutil "qfw/mongodb"
@@ -299,7 +300,46 @@ func sendmail(appid string, days int, force int, fn string, bcron bool, startTim
 				data, bdata := Mgo.Find("usermail", dataQuery, `{"_id":1}`, fields2, false, -1, -1)
 				firstId, lastId := "", ""
 				if bdata && data != nil && *data != nil && len(*data) > 0 {
-					infos := *data
+					infos := []map[string]interface{}{}
+					//数据去重
+					if cronday == 0 {
+						log.Println("需要走数据去重 ", appid)
+						datasMap := map[string]bool{}
+						sdata, sok := Mgo.Find("userdatalog", map[string]interface{}{"appid": appid, "flag": "sendmail"}, `{"_id":-1}`, nil, false, 0, 1)
+						if sok && sdata != nil && len(*sdata) > 0 {
+							sfirstId := qu.ObjToString((*sdata)[0]["first"])
+							slastId := qu.ObjToString((*sdata)[0]["last"])
+							log.Println("起始--结束 ", sfirstId, slastId)
+							udata, uok := Mgo.Find("usermail", map[string]interface{}{
+								"appid": appid, "_id": map[string]interface{}{
+									"$gte": mongodb.StringTOBsonId(sfirstId),
+									"$lte": mongodb.StringTOBsonId(slastId),
+								},
+							}, nil, nil, false, -1, -1)
+							if uok && udata != nil && len(*udata) > 0 {
+								log.Println("重复数据数量 ", len(*udata), appid)
+								for _, u := range *udata {
+									uid := qu.ObjToString(u["id"])
+									datasMap[uid] = true
+								}
+							} else {
+								log.Println("无重复数据数量", appid, uok, udata)
+							}
+						}
+						for _, d := range *data {
+							infoid := qu.ObjToString(d["id"])
+							if !datasMap[infoid] {
+								datasMap[infoid] = true
+								infos = append(infos, d)
+							} else {
+								log.Println("过滤数据 ", infoid)
+							}
+						}
+					} else {
+						log.Println("不走数据去重 ", appid, cronday)
+						infos = *data
+					}
+					//
 					allUserData := map[string][]map[string]interface{}{}
 					updateMap := [][]map[string]interface{}{}
 					newGetLen := 0
@@ -412,7 +452,7 @@ func sendmail(appid string, days int, force int, fn string, bcron bool, startTim
 								}
 							}
 						} else {
-							log.Println("未分发 ", items)
+							// log.Println("未分发 ", items)
 							userMap := allUserData[item]
 							if userMap == nil {
 								userMap = []map[string]interface{}{}