Merge branch 'master' into feature/v4.9.3

src/config.json

@@ -452,6 +452,12 @@
   "ipInitAuthentication": "qwertyuiopzxcvbnm1234567890asdfghjkl",
   "node": 1,
   "bootstrap_specification": "剑鱼工作台是剑鱼注册用户的统一访问入口。通过剑鱼工作台，用户可以访问到剑鱼平台提供的所有数据、资讯、情报、业务功能和各类服务。剑鱼工作台为个人用户提供方便快捷、集约高效的工作环境，同时也为企业用户提供业情一体、规范管理和安全可靠的数字化营销环境。",
+  "anonymousLimit": {
+    "switch": true,
+    "times": 5,
+    "second": 300,
+    "limit": 36000
+  },
   "nzjLimit": 100,
   "nzjConstructionContent": "拟在建项目是指尚在规划、报批、立项、环评、设计等阶段的项目，一般由发改委和各部委提供，且项目规模和投资额都较大。剑鱼标讯一般会在项目正式招标前3-12个月就通知付费客户，让客户有足够的时间和空间和业主联系，做好前期准备工作。",
   "nzjProgressRemark": "进展备注是剑鱼标讯通过一定渠道获得的项目进度计划，帮您提前把握商机！开通大会员即可享受此权益。"

src/jfw/config/config.go

@@ -6,12 +6,11 @@ import (
 	"app.yhyue.com/moapp/jybase/mail"
 	"app.yhyue.com/moapp/jypkg/middleground"
 	"app.yhyue.com/moapp/jypkg/public"
-	"fmt"
 	"github.com/gogf/gf/v2/frame/g"
 	"github.com/gogf/gf/v2/os/gcfg"
 	"github.com/gogf/gf/v2/os/gctx"
-	"github.com/mozillazg/go-pinyin"
-	"strings"
+	"github.com/gogf/gf/v2/util/gconv"
+	"log"
 	"sync"
 )
 
@@ -32,6 +31,7 @@ var (
 	IndustryTrendConfig  []IndustryTrend
 	IpList               *ipmatch.WhiteIp
 	IpInitLock           sync.RWMutex //锁
+	JyAnonymousLimit     *anonymousLimit
 )
 
 type IndustryTrend struct {
@@ -57,6 +57,14 @@ type entnichePcConf struct {
 	PrivateDataPhone string `json:"privateDataPhone"`
 }
 
+// AnonymousLimit 匿名用户访问限制
+type anonymousLimit struct {
+	Switch bool  `json:"switch"` //是否开启
+	Times  int64 `json:"times"`  //请求多少次
+	Second int   `json:"second"` //时间范围
+	Limit  int   `json:"limit"`  //限制多长时间
+}
+
 func init() {
 	g.Cfg().GetAdapter().(*gcfg.AdapterFile).SetFileName("config.yaml")
 	util.ReadConfig(&Sysconfig)
@@ -85,6 +93,10 @@ func init() {
 	var ctx = gctx.New()
 	// 获取配置数据
 	_ = g.Cfg("index").MustGet(ctx, "industryArr").Structs(&IndustryTrendConfig)
+	JyAnonymousLimit = &anonymousLimit{}
+	if err := gconv.Struct(Sysconfig["anonymousLimit"], JyAnonymousLimit); err != nil {
+		log.Println(err)
+	}
 
 	Middleground = middleground.NewMiddleground(g.Cfg().MustGet(ctx, "etcd.hosts").Strings()).
 		RegUserCenter(g.Cfg().MustGet(ctx, "userCenterKey").String()).
@@ -114,105 +126,4 @@ func IpInit() {
 	IpList = ipmatch.NewRb(nil)
 }
 
-func GetLetterMap(names string) []map[string]interface{} {
-	var (
-		data                                   []map[string]interface{}
-		nameArr1, nameArr2, nameArr3, nameArr4 []string
-	)
-	nameArr := strings.Split(names, ",")
-	for _, v := range nameArr {
-		nameArr1 = append(nameArr1, fmt.Sprintf(`"%s"`, v))
-	}
-	dataArr2 := public.BaseMysql.SelectBySql(fmt.Sprintf(`select id,name,letter from seo_words.seo_resource where name in (%s) and state1=2 order by id desc`, strings.Join(nameArr1, ",")))
-	dataArr1 := public.BaseMysql.SelectBySql(fmt.Sprintf(`select a.id,a.name,b.id class_id,b.name class_1 from seo_words.seo_industry a inner join  seo_words.seo_industry_class b on a.state=1 and  a.class_1 in (%s) and a.class_1=b.name and  a.class_2 !='药品'  order by a.class_1`, strings.Join(nameArr1, ",")))
-	for _, v1 := range nameArr {
-		var isNormal bool
-		if dataArr1 != nil && len(*dataArr1) > 0 {
-			for _, v2 := range *dataArr1 {
-				if v1 == v2["name"] {
-					isNormal = true
-					data = append(data, map[string]interface{}{
-						"name": v1 + util.ObjToString(Seoconfig["seoKeywordSuffix"]),
-						"url":  fmt.Sprintf("/tags/industry/%v_all_all_%v.html", util.Int64All(v2["class_id"]), util.Int64All(v2["id"])),
-					})
-				}
-			}
-		}
-		if !isNormal {
-			if dataArr2 != nil && len(*dataArr2) > 0 {
-				for _, v2 := range *dataArr2 {
-					if v1 == v2["name"] {
-						isNormal = true
-						letter := util.ObjToString(v2["letter"])
-						//判断问题
-						id := util.Int64All(v2["id"])
-						v1 += util.ObjToString(Seoconfig["seoKeywordSuffix"])
-						data = append(data, map[string]interface{}{
-							"name": v1,
-							"url":  fmt.Sprintf("/tags/letter/all_all_all_%v_%v.html", letter, id),
-						})
-					}
-				}
-			}
-		}
-		if !isNormal {
-			nameArr2 = append(nameArr2, v1)
-			nameArr3 = append(nameArr3, fmt.Sprintf(`"%s"`, v1))
-			letter := getFirstChar(v1)
-			nameArr4 = append(nameArr4, fmt.Sprintf(`("%s", "%s", 2)`, v1, letter))
-		}
-	}
-	if len(nameArr2) > 0 {
-		public.BaseMysql.SelectBySql(fmt.Sprintf(`INSERT INTO seo_words.seo_resource (name, letter, state1) VALUES %s`, strings.Join(nameArr4, ",")))
-		dataArr3 := public.BaseMysql.SelectBySql(fmt.Sprintf(`select id,name,letter from seo_words.seo_resource where name in (%s) and state1=2 order by id desc`, strings.Join(nameArr3, ",")))
-		if dataArr3 != nil {
-			for _, v3 := range *dataArr3 {
-				letter := util.ObjToString(v3["letter"])
-				id := util.Int64All(v3["id"])
-				n := util.ObjToString(v3["name"])
-				n += util.ObjToString(Seoconfig["seoKeywordSuffix"])
-				data = append(data, map[string]interface{}{
-					"name": n,
-					"url":  fmt.Sprintf("/tags/letter/all_all_all_%v_%v.html", letter, id),
-				})
-			}
-		}
-	}
-	return data
-}
-
 var DigitalEnglish = map[string]string{"0": "L", "1": "Y", "2": "E", "3": "S", "4": "S", "5": "W", "6": "L", "7": "Q", "8": "B", "9": "J"}
-
-// 获取单词的首字母大写
-func getFirstChar(word string) string {
-	firstChar := string(word[0])
-
-	// 检查第一个字符是否是英文字母
-	if isEnglishLetter(firstChar) {
-		return strings.ToUpper(firstChar)
-	}
-	// 检查第一个字符是否是数字
-	if v, ok := DigitalEnglish[firstChar]; ok && v != "" {
-		return v
-	}
-	// 转换为拼音并获取首字母大写
-	py := convertToPinyin(word)
-	return strings.ToUpper(string(py[0]))
-}
-
-// 判断字符是否是英文字母
-func isEnglishLetter(char string) bool {
-	return len(char) == 1 && ((char >= "a" && char <= "z") || (char >= "A" && char <= "Z"))
-}
-
-// 将中文单词转换为拼音
-func convertToPinyin(word string) string {
-	p := pinyin.NewArgs()
-	p.Style = pinyin.FirstLetter
-	pys := pinyin.Pinyin(word, p)
-	pinyinStr := ""
-	for _, py := range pys {
-		pinyinStr += py[0]
-	}
-	return pinyinStr
-}

src/jfw/filter/anonymousUser.go

@@ -4,6 +4,7 @@ import (
 	util "app.yhyue.com/moapp/jybase/common"
 	"app.yhyue.com/moapp/jybase/encrypt"
 	"app.yhyue.com/moapp/jybase/go-xweb/httpsession"
+	"app.yhyue.com/moapp/jybase/redis"
 	"app.yhyue.com/moapp/jypkg/public"
 	"crypto/rand"
 	"crypto/rsa"
@@ -15,6 +16,7 @@ import (
 	"fmt"
 	"github.com/bwmarrin/snowflake"
 	"io/ioutil"
+	"jy/src/jfw/config"
 	"log"
 	"net/http"
 	"strings"
@@ -30,7 +32,8 @@ type AnonymousAuth struct {
 }
 
 var (
-	Node *snowflake.Node
+	Node     *snowflake.Node
+	saveChan = make(chan map[string]interface{}, 10000)
 )
 
 func initPrivatePublicKey() {
@@ -93,9 +96,95 @@ func initPrivatePublicKey() {
 	PublicKey = rsaPublicKey
 }
 
-func (sk *AnonymousAuth) Do() {
+func ChanMonitor() {
+
+	var (
+		saveData []map[string]interface{}
+		upData   []map[string]interface{}
+		saveKey  = []string{"ip", "client", "os", "browse", "url", "guestUID", "mdescribe", "refer", "method", "creation_time"}
+		count    int
+	)
+	for {
+		select {
+		case data := <-saveChan:
+			count++
+			if data["type"] == 1 { //插入
+				saveData = append(saveData, data)
+			} else if data["type"] == 2 { //更新
+				upData = append(upData, data)
+			}
+			// 处理 100条时处理一次 saveChan 收到的数据
+			if count == 100 {
+				if len(saveData) > 0 {
+					var ss []interface{}
+					for _, datum := range saveData {
+						for _, s := range saveKey {
+							ss = append(ss, datum[s])
+						}
+					}
+					public.BaseMysql.InsertBatch("anonymous_identity", saveKey, ss)
+					saveData = []map[string]interface{}{}
+				}
+				// 处理 upChan 收到的数据
+				if len(upData) > 0 {
+					var (
+						ids              []string
+						column1, column2 []string
+					)
+					for _, i := range upData {
+						//upChan <- map[string]interface{}{"fid":| fid.Value, "trustedId": trustedId, "upId": jyGuestUID.Value}
+						jyGuestUID := util.InterfaceToStr(i["upId"])
+						ids = append(ids, fmt.Sprintf(`"%s"`, jyGuestUID))
+						column1 = append(column1, fmt.Sprintf(` WHEN guestUID = '%s' THEN '%s' `, jyGuestUID, util.InterfaceToStr(i["fid"])))
+						column2 = append(column2, fmt.Sprintf(` WHEN guestUID = '%s' THEN '%s' `, jyGuestUID, util.InterfaceToStr(i["trustedId"])))
+					}
+
+					public.BaseMysql.SelectBySql(fmt.Sprintf(`UPDATE anonymous_identity
+                    SET fid = CASE %s ELSE fid END,
+                        trustedId = CASE %s ELSE trustedId END
+                    WHERE guestUID IN (%s);`, strings.Join(column1, " "), strings.Join(column2, " "), strings.Join(ids, " ")))
+					upData = []map[string]interface{}{}
+				}
+				count = 0
+			}
+
+		case <-time.After(30 * time.Second):
+			if len(saveData) > 0 {
+				var ss []interface{}
+				for _, datum := range saveData {
+					for _, s := range saveKey {
+						ss = append(ss, datum[s])
+					}
+				}
+				public.BaseMysql.InsertBatch("anonymous_identity", saveKey, ss)
+				saveData = []map[string]interface{}{}
+			}
+			if len(upData) > 0 {
+				var (
+					ids              []string
+					column1, column2 []string
+				)
+				for _, i := range upData {
+					//upChan <- map[string]interface{}{"fid":| fid.Value, "trustedId": trustedId, "upId": jyGuestUID.Value}
+					jyGuestUID := util.InterfaceToStr(i["upId"])
+					ids = append(ids, fmt.Sprintf(`"%s"`, jyGuestUID))
+					column1 = append(column1, fmt.Sprintf(` WHEN guestUID = '%s' THEN '%s' `, jyGuestUID, util.InterfaceToStr(i["fid"])))
+					column2 = append(column2, fmt.Sprintf(` WHEN guestUID = '%s' THEN '%s' `, jyGuestUID, util.InterfaceToStr(i["trustedId"])))
+				}
+
+				public.BaseMysql.SelectBySql(fmt.Sprintf(`UPDATE anonymous_identity
+                    SET fid = CASE %s ELSE fid END,
+                        trustedId = CASE %s ELSE trustedId END
+                    WHERE guestUID IN (%s);`, strings.Join(column1, " "), strings.Join(column2, " "), strings.Join(ids, ",")))
+				upData = []map[string]interface{}{}
+			}
+		}
+	}
+}
+
+func (sk *AnonymousAuth) Do() bool {
 	if sk.GetSession["mgoUserId"] != nil && sk.GetSession["mgoUserId"] != "" {
-		return
+		return true
 	}
 	jyTrustedId, _ := sk.R.Cookie("JYTrustedId")
 	if jyTrustedId == nil || jyTrustedId.Value == "" { //不存在信用标识
@@ -103,8 +192,14 @@ func (sk *AnonymousAuth) Do() {
 		if jyGuestUID == nil || jyGuestUID.Value == "" { //不存在后端id 生成后端id 并创建匿名身份信息
 			accept := sk.R.Header.Get("Accept")
 			if sk.R.Method != "GET" || !strings.Contains(accept, "text/html") { //避免多请求
-				return
+				return true
+			}
+			//增加限制未携带唯一标识的请求频率
+			if !sk.noLoginLimit(sk.R, util.If(len(sk.GetSession) > 1, string(sk.Session.Id()), "").(string)) {
+				sk.W.WriteHeader(502)
+				return false
 			}
+
 			guestUID := Node.Generate()
 			cookie := &http.Cookie{
 				Name:     "JYGuestUID",
@@ -130,8 +225,10 @@ func (sk *AnonymousAuth) Do() {
 				"refer":         sk.R.Referer(),
 				"method":        sk.R.Method,
 				"creation_time": time.Now().Unix(),
+				"type":          1,
 			}
-			public.BaseMysql.Insert("anonymous_identity", data)
+			saveChan <- data
+			//public.BaseMysql.Insert("anonymous_identity", data)
 			//stmt, err := config.ClickhouseDb.Prepare("INSERT INTO anonymous_identity (ip, client, os, browse, url, guestUID, mdescribe, refer, method, creation_time) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
 			//if err != nil {
 			//	return
@@ -141,25 +238,25 @@ func (sk *AnonymousAuth) Do() {
 			//if err != nil {
 			//	return
 			//}
-			return
+			return true
 		}
 
 		fid, err := sk.R.Cookie("fid")
 		if err != nil {
 			//log.Println("匿名用户获取fid失败", err.Error())
-			return
+			return true
 		}
 		eid, err := sk.R.Cookie("eid")
 		if err != nil {
-			return
+			return true
 		}
 		if fid.Value != "" && eid.Value != "" {
 			eData, err1 := Decryption(eid.Value)
 			if err1 != nil {
-				return
+				return true
 			}
 			if fid.Value != eData {
-				return
+				return true
 			}
 			trustedId := encrypt.SE.EncodeString(fid.Value + "***" + jyGuestUID.Value)
 			cookie := &http.Cookie{
@@ -171,8 +268,9 @@ func (sk *AnonymousAuth) Do() {
 				Expires: time.Now().AddDate(10, 0, 0),
 			}
 			http.SetCookie(sk.W, cookie)
-			public.BaseMysql.Update("anonymous_identity", map[string]interface{}{"guestUID": jyGuestUID.Value},
-				map[string]interface{}{"fid": fid.Value, "trustedId": trustedId})
+			saveChan <- map[string]interface{}{"fid": fid.Value, "trustedId": trustedId, "upId": jyGuestUID.Value, "type": 2}
+			//public.BaseMysql.Update("anonymous_identity", map[string]interface{}{"guestUID": jyGuestUID.Value},
+			//	map[string]interface{}{"fid": fid.Value, "trustedId": trustedId})
 			//stmt, err := config.ClickhouseDb.Prepare("UPDATE anonymous_identity SET fid = ? and trustedId = ? WHERE guestUID = ?")
 			//if err != nil {
 			//	fmt.Println("Failed to prepare statement:", err)
@@ -192,6 +290,7 @@ func (sk *AnonymousAuth) Do() {
 			log.Println("无效信用id")
 		}
 	}
+	return true
 }
 
 // Encryption  加密
@@ -230,3 +329,38 @@ func Decryption(content string) (string, error) {
 	}
 	return string(decryptedText), nil
 }
+
+func (sk *AnonymousAuth) noLoginLimit(r *http.Request, sessionId string) bool {
+	if !config.JyAnonymousLimit.Switch {
+		return true
+	}
+	var ip string
+	for _, t_ip := range strings.Split(util.GetIp(r), ",") {
+		if len(t_ip) > 0 && t_ip != "127.0.0.1" {
+			ip = t_ip
+			break
+		}
+	}
+	if len(ip) > 0 && config.IpList.Match(ip) { //百度白名单
+		return true
+	}
+
+	limitKey := fmt.Sprintf("anonymousLimit_Id_%s", sessionId)
+	if sessionId == "" {
+		limitKey = fmt.Sprintf("anonymousLimit_Ip_%s", ip)
+	}
+	if redis.Get("limitation", fmt.Sprintf("%s_limit", limitKey)) != nil {
+		return false
+	}
+	val := redis.Incr("limitation", limitKey)
+	if val == 1 { //设置过期值
+		if err := redis.SetExpire("limitation", limitKey, config.JyAnonymousLimit.Second); err != nil {
+			log.Printf("noLoginLimit SetExpire key:%s err:%v ", limitKey, err.Error())
+		}
+	}
+	if val > config.JyAnonymousLimit.Times {
+		redis.Put("limitation", fmt.Sprintf("%s_limit", limitKey), 1, config.JyAnonymousLimit.Limit)
+		return false
+	}
+	return true
+}

src/jfw/filter/filter.go

@@ -28,6 +28,7 @@ var (
 func init() {
 	xweb.AddFilter(&Filter{})
 	go initPrivatePublicKey()
+	go ChanMonitor()
 	Node, _ = snowflake.NewNode(util.Int64All(Seoconfig["node"]))
 	//日志过滤 路由集合
 	go fs.FSNotifyFUNC("sword->%s", "./route.json", true, func() {
@@ -54,7 +55,9 @@ func (f *Filter) Do(w http.ResponseWriter, r *http.Request) bool {
 	if getSession["base_user_id"] != nil && getSession["positionId"] == nil {
 		identity.SwitchToBest(util.Int64All(getSession["base_user_id"]), session, Middleground, &public.MQFW, false)
 	}
-	(&AnonymousAuth{w, r, session, getSession, make(map[string]interface{})}).Do()
+	if !(&AnonymousAuth{w, r, session, getSession, make(map[string]interface{})}).Do() {
+		return false
+	}
 	(&CookieInfo{w, r}).Do()
 
 	if !(&logFilter{w, r, session, getSession, make(map[string]interface{})}).Do() {