feat:html标签清理

go.mod

@@ -6,7 +6,7 @@ require (
 	app.yhyue.com/moapp/jybase v0.0.0-20220419023055-f406279ff7e3
 	bp.jydev.jianyu360.cn/BP/jynsq v0.0.0-20220222052708-ebc43af90698
 	bp.jydev.jianyu360.cn/BaseService/gateway v0.0.0-20220419090715-88ddb32961be
-	bp.jydev.jianyu360.cn/BaseService/resourceCenter v0.0.0-20220419063004-233fc7ce006c
+	github.com/PuerkitoBio/goquery v1.8.0
 	github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible
 	github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
 	github.com/go-sql-driver/mysql v1.6.0

go.sum

@@ -1,7 +1,6 @@
 app.yhyue.com/moapp/esv1 v0.0.0-20220414031211-3da4123e648d h1:WPsYuuptAd3UEgN+jPzpnsDe/OvcshDUUtOTZPYGSJ8=
 app.yhyue.com/moapp/esv1 v0.0.0-20220414031211-3da4123e648d/go.mod h1:91/lSD/hS+ckMVP3WdidRzDhC60lLMdyce9QHy0cSMA=
 app.yhyue.com/moapp/jybase v0.0.0-20220415064050-37ce64b3e2d4/go.mod h1:qNRA0sHuYqcLoYoP8irpaWnW9YsXixe6obBIkwaXpD0=
-app.yhyue.com/moapp/jybase v0.0.0-20220418104200-46c3fff161c7/go.mod h1:qNRA0sHuYqcLoYoP8irpaWnW9YsXixe6obBIkwaXpD0=
 app.yhyue.com/moapp/jybase v0.0.0-20220419023055-f406279ff7e3 h1:VTcLmCQLSBr9SUDCsvxXiRENOgTAt9EN9CPwv+Lg8mA=
 app.yhyue.com/moapp/jybase v0.0.0-20220419023055-f406279ff7e3/go.mod h1:qNRA0sHuYqcLoYoP8irpaWnW9YsXixe6obBIkwaXpD0=
 bp.jydev.jianyu360.cn/BP/jynsq v0.0.0-20220222052708-ebc43af90698 h1:2xGDnrmbGt4VzAmDSsKH+C9Bq40z9eQ4NJfH/779o8g=
@@ -9,8 +8,6 @@ bp.jydev.jianyu360.cn/BP/jynsq v0.0.0-20220222052708-ebc43af90698/go.mod h1:ojo/
 bp.jydev.jianyu360.cn/BaseService/gateway v0.0.0-20220419090715-88ddb32961be h1:YCPWJEM8JHtV5ush0OEB85mKKsn2hpzKNmFhnbXyMRY=
 bp.jydev.jianyu360.cn/BaseService/gateway v0.0.0-20220419090715-88ddb32961be/go.mod h1:Yj4oabIGItuMoF0BXYLz2XAnF581kxgXBrvlUtIJrkI=
 bp.jydev.jianyu360.cn/BaseService/resourceCenter v0.0.0-20220418005748-8ba5d936dd53/go.mod h1:E5lcDI3k4FESLxiAetCfWQTq8qfpy9cv0yN1oKoEO34=
-bp.jydev.jianyu360.cn/BaseService/resourceCenter v0.0.0-20220419063004-233fc7ce006c h1:P0B4xYJOJUVWcuWZKFqKlMcrJ1s0/GhvTBEkrJT+KSY=
-bp.jydev.jianyu360.cn/BaseService/resourceCenter v0.0.0-20220419063004-233fc7ce006c/go.mod h1:6KL5LMEku83uRbre0W/bj5kXG2I6pJGBFtktmtp51yM=
 bp.jydev.jianyu360.cn/BaseService/userCenter v0.0.0-20220418072311-2062bed1e700/go.mod h1:KjcrxTzM96tBc6G4B8tlLBn1lrVy5UJYF8+eTdP4xAE=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
@@ -62,6 +59,8 @@ github.com/ClickHouse/clickhouse-go v1.5.1/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHg
 github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/Shopify/sarama v1.30.0/go.mod h1:zujlQQx1kzHsh4jfV1USnptCQrHAEZ2Hk8fTKCulPVs=
@@ -78,6 +77,8 @@ github.com/alicebob/miniredis/v2 v2.17.0/go.mod h1:gquAfGbzn92jvtrSC69+6zZnwSODV
 github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible h1:9gWa46nstkJ9miBReJcN8Gq34cBFbzSpQZVVT9N09TM=
 github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
 github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
@@ -654,6 +655,7 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=

rpc/common/test/html_test.go

@@ -8,7 +8,7 @@ import (
 
 //html
 func Test_html(t *testing.T) {
-	str1 := "<script>alert(2)</script>"
+	str1 := "<script>中国人</script>"
 	str2 := html.EscapeString(str1)
 	fmt.Println(str2)
 	str3 := html.UnescapeString(str2)

rpc/consumer/internal/logic/publishinfologic.go

@@ -7,7 +7,6 @@ import (
 	"app.yhyue.com/moapp/jyInfo/rpc/util"
 	"context"
 	"fmt"
-	"html"
 	"log"
 	"strings"
 	"time"
@@ -46,7 +45,7 @@ func (l *PublishInfoLogic) PublishInfo(in *consumerinfo.PublishInfoReq) (*consum
 	publishData["title"] = in.Title                                       //信息标题
 	publishData["province"] = in.Province                                 //项目省份
 	publishData["city"] = in.City                                         //项目城市
-	publishData["detail"] = html.EscapeString(in.Detail)                  //正文信息
+	publishData["detail"] = util.NewCut().ClearHtml(in.Detail)            //正文信息
 	publishData["create_time"] = time.Now().Format("2006-01-02 15:04:05") //申请时间
 	publishData["attach"] = in.Attach                                     //附件(多个附件逗号分割)
 	publishData["contact_person"] = in.Contact.Person                     //联系人

rpc/manager/go_build_manager_go_linux_linux

@@ -1 +0,0 @@
-ELF

rpc/manager/internal/logic/infoonekeyactionlogic.go

@@ -13,6 +13,7 @@ import (
 	"context"
 	"fmt"
 	"github.com/zeromicro/go-zero/core/logx"
+	"log"
 	"time"
 )
 
@@ -36,7 +37,8 @@ func (l *InfoOneKeyActionLogic) InfoOneKeyAction(in *manager.OneKeyActionReq) (*
 	if in.UserId != "" && MI.MUserIdMap[in.UserId] {
 		msgId := se.SE.DecodeString(in.MsgId)
 		var tname = mc.If(in.MsgType == "3", "supply_info", "information").(string)
-		data := model.Mysql.SelectBySql(`SELECT a.title,a.detail,a.attach FROM ? a WHERE a.id = ? AND a.type = ?`, tname, msgId, in.MsgType)
+		data := model.Mysql.SelectBySql(fmt.Sprintf(`SELECT title,detail,attach FROM %s  WHERE id = ? AND type = ?`, tname), msgId, in.MsgType)
+		log.Println("---:", *data)
 		if len(*data) > 0 {
 			res := (*data)[0]
 			switch in.Type {

rpc/util/htmlclear.go

@@ -0,0 +1,166 @@
+package util
+
+import (
+	mc "app.yhyue.com/moapp/jybase/common"
+	"github.com/PuerkitoBio/goquery"
+	"regexp"
+	"strings"
+	"unicode/utf8"
+)
+
+//
+type Cut struct {
+	annotate      *regexp.Regexp
+	tag           *regexp.Regexp
+	scripttag     *regexp.Regexp
+	inputag       *regexp.Regexp
+	isborder      *regexp.Regexp
+	hiddentag     *regexp.Regexp
+	styletag      *regexp.Regexp
+	colstag       *regexp.Regexp
+	rowstag       *regexp.Regexp
+	display       *regexp.Regexp
+	multiCR       *regexp.Regexp
+	replBlankLine *regexp.Regexp
+	replStartWrap *regexp.Regexp
+	replTags2CR   []string
+	retainTags2CR []string
+}
+
+//
+func NewCut() *Cut {
+	t, _ := regexp.Compile("<[^>]+>")
+	m, _ := regexp.Compile("([\r\n][\u3000\u2003\u00a0\\s]*)+|[\r\n]+")
+	//sc, _ := regexp.Compile("\\<script[^\\>]*\\>*[^\\>]+\\</script\\>")
+	//ss, _ := regexp.Compile("\\<style[^\\>]*\\>*[^\\>]+\\</style\\>")
+	scs := regexp.MustCompile("(?s)<(script|style)[^>]*>.+?</(script|style)>")
+	at := regexp.MustCompile("(?s)<(!%-%-|!--).*?(%-%-|--)>") //注释 css
+	hiddentag := regexp.MustCompile(`<\s*input[^<]*type=("|')hidden("|')[^<]*>`)
+	input := regexp.MustCompile(`<\s*input[^<]*value=("|')([^>"']*)[^<]*>`)
+	cols, _ := regexp.Compile(`colspan="\d+"`)
+	rows, _ := regexp.Compile(`rowspan="\d+"`)
+	border, _ := regexp.Compile(`(border="(\d+)")|(cellpadding="(\d+)")|(cellspacing="(\d+)")`)
+	dis, _ := regexp.Compile(`display:none`)
+	return &Cut{
+		annotate:      at,
+		tag:           t,
+		scripttag:     scs,
+		hiddentag:     hiddentag,
+		inputag:       input,
+		colstag:       cols,
+		isborder:      border,
+		rowstag:       rows,
+		display:       dis,
+		multiCR:       m,
+		replBlankLine: regexp.MustCompile("\\s+[\r\n]"),
+		replStartWrap: regexp.MustCompile("^[\u3000\u2003\u00a0\\s]+|[\u3000\u2003\u00a0\\s]+$"),
+		replTags2CR:   []string{"div", "p", "br", "h1", "h2", "h3", "h4", "h5"},
+		retainTags2CR: []string{"table", "thead", "tfoot", "tbody", "th", "td", "tr"},
+	}
+}
+
+//清理HTML标签
+func (c *Cut) ClearHtml(src string) string {
+	src = c.replBlankLine.ReplaceAllString(src, "")
+	src = strings.Replace(src, ">\n", ">", -1)
+	src = strings.Replace(src, " ", "", -1)
+	//标签全转小写
+	src = c.tag.ReplaceAllStringFunc(src, strings.ToLower)
+	//清script,style
+	src = c.scripttag.ReplaceAllString(src, "")
+	//清理注释文本
+	src = c.annotate.ReplaceAllString(src, "")
+	//清理input
+	src = c.hiddentag.ReplaceAllString(src, "")
+	src = c.inputag.ReplaceAllString(src, "$2")
+	document, err := goquery.NewDocumentFromReader(strings.NewReader(src))
+	if err == nil {
+		if tmpstr, err := document.Each(func(i int, sel *goquery.Selection) {
+			sel.Find("td").Each(func(i int, selection *goquery.Selection) {
+				val, b := selection.Attr("title")
+				if b && strings.Trim(val, " ") != "" {
+					tmpstr := strings.TrimFunc(selection.Text(), func(r rune) bool {
+						return r == 9 || r == 32
+					})
+					if utf8.RuneCountInString(strings.Trim(tmpstr, " ")) < utf8.RuneCountInString(strings.Trim(val, " ")) {
+						selection.SetText(strings.Trim(val, " "))
+					}
+				}
+			})
+		}).Html(); err == nil {
+			src = tmpstr
+		}
+	}
+	//换结束标签
+	src = c.tag.ReplaceAllStringFunc(src, func(tmp string) string {
+		tmp = strings.Replace(tmp, " ", "", -1)
+		//保留这些标签
+		for _, v := range c.retainTags2CR {
+			if "<"+v+">" == tmp || "</"+v+">" == tmp {
+				if tmp == "</table>" {
+					return tmp + "\n"
+				}
+				return tmp
+			}
+			if strings.HasPrefix(tmp, "<"+v) {
+				dispstrs := c.display.FindAllString(tmp, -1)
+				rowstrs := c.rowstag.FindAllString(tmp, -1)
+				colstrs := c.colstag.FindAllString(tmp, -1)
+				con := "<" + v
+				if con == "<table" {
+					if isHasBoder(tmp, c.isborder) {
+						con = con + ` border="1"`
+					}
+				}
+				if len(colstrs) > 0 { //处理多列合并
+					con += " " + colstrs[0]
+				}
+				if len(rowstrs) > 0 { //处理多行合并
+					con += " " + rowstrs[0]
+				}
+				if len(dispstrs) > 0 {
+					con += " style=\"" + dispstrs[0] + "\""
+				}
+				return con + ">"
+
+			}
+		}
+		if tmp == "<br>" || tmp == "<br/>" || tmp == "<center>" || tmp == "</center>" || tmp == "<ul>" || tmp == "</ul>" {
+			return "\n"
+		}
+		if tmp[1] != 47 { //开始标签
+			for _, v := range c.replTags2CR {
+				if v == tmp[1:len(tmp)-1] {
+					return "\n"
+				}
+			}
+			return ""
+		}
+		for _, v := range c.replTags2CR {
+			if v == tmp[2:len(tmp)-1] {
+				return "\n"
+			}
+		}
+		return ""
+	})
+	src = c.replStartWrap.ReplaceAllString(src, "")
+	src = c.replBlankLine.ReplaceAllString(src, "\n")
+	//清除多余换行
+	c.multiCR.ReplaceAllString(src, "\n")
+	return strings.Replace(src, "\n", "<br/>", -1)
+}
+
+//判断table是否加表格线
+func isHasBoder(con string, reg *regexp.Regexp) bool {
+	res := reg.FindAllStringSubmatch(con, -1)
+	hasBorder := false
+	for _, v := range res {
+		for k, val := range v {
+			if k > 0 && k%2 == 0 && mc.IntAll(val) > 0 {
+				hasBorder = true
+				break
+			}
+		}
+	}
+	return hasBorder
+}