优化附件下载和公共方法

src/res/util/comm.lua

@@ -517,6 +517,7 @@ function common.getFilesLinkByTag(href,tags,content,withend)
 	--dhtml=dhtml.."<a href='/123.doc'>123.doc</a>"
 	local alist = findListHtml(tags.." a", content)
 	local flist={}
+	local ahref={}--对href进行判重处理
 	for k,v in pairs(alist) do
 		local item={}
 		item["href"]="a:eq("..tostring(k-1).."):attr(href)"
@@ -550,8 +551,16 @@ function common.getFilesLinkByTag(href,tags,content,withend)
 		end
 	
 		if statehref~=nil and item["title"]~="" then
-			
-			table.insert(flist,item)
+			local has = false
+			for _,tmphref in pairs(ahref) do
+				if tmphref == item["href"] then
+					has = true
+				end
+			end
+			if not has then
+				table.insert(ahref,item["href"])
+				table.insert(flist,item)
+			end
 		end
 	end
 	return flist
@@ -568,7 +577,6 @@ function common.getFileAttachmentsArrayWithTag(href,tags,content,withend,param,h
 	--local nameTypeArr={"jpg","JPG","bid","pdf","PDF","doc","docx","xls","xlsx","zip","rar","swf","DOCX","DOC","PDF","XLS","XLSX","ZIP","RAR","SWF"}
 	local titleAndHrefList = common.getFilesLinkByTag(href,tags,content,withend)
 	for i,v in ipairs(titleAndHrefList) do
-		
 		local end_type = string.find(v["title"],v["ftype"].."$")
 		local file_name = ""
 		if end_type==nil or end_type=="" then
@@ -581,13 +589,13 @@ function common.getFileAttachmentsArrayWithTag(href,tags,content,withend,param,h
 		local init_url = v["href"]
 		if url == "" then
 			local u = 0
-			while u < 6 do
+			while u < 2 do
 				u = u + 1
 				url,name,size,ftype,fid=downloadFile(file_name,v["href"],"get",param,head,ck)
 				if url ~= "" and size ~= "" then
-					u = 7   -- 下载无误 跳出循环
+					u = 3   -- 下载无误 跳出循环
 				end
-				if u==6 and (url == "" or size == "") then
+				if u==2 and (url == "" or size == "") then
 					saveErrLog(v["href"],"comm附件下载失败")
 				end
 			end

src/spider/script.go

@@ -608,6 +608,34 @@ func (s *Script) LoadScript(code, script_file string, newstate bool) string {
 		S.Push(lua.LString(publishtime))
 		return 1
 	}))
+	//匹配
+	s.L.SetGlobal("stringFind", s.L.NewFunction(func(S *lua.LState) int {
+		regstr := S.ToString(-1)
+		text := S.ToString(-2)
+		reg := regexp.MustCompile(regstr)
+		result := reg.FindString(text)
+		isMatch := false
+		if result != "" {
+			isMatch = true
+		}
+		S.Push(lua.LString(result))
+		S.Push(lua.LBool(isMatch))
+		return 2
+	}))
+	//去除特殊标签中间内容
+	s.L.SetGlobal("getPureContent", s.L.NewFunction(func(S *lua.LState) int {
+		con := S.ToString(-1)
+		reg := regexp.MustCompile("(?s)<!%-%-.*?%-%->")
+		con = reg.ReplaceAllString(con, "")
+		// indexArr := reg.FindAllStringIndex(con, -1)
+		// for i := len(indexArr) - 1; i >= 0; i-- {
+		// 	if index := indexArr[i]; len(index) == 2 {
+		// 		con = con[:index[0]] + con[index[1]:]
+		// 	}
+		// }
+		S.Push(lua.LString(con))
+		return 1
+	}))
 	return ""
 }
 func dealHref(pageListUrl, href string) string {