chrome下载方法修改；公共方法修改

src/spider/download.go

@@ -19,7 +19,7 @@ import (
 	"github.com/surfer/agent"
 )
 
-var regImgStr = "\\.(JPG|jpg|GIF|gif|PNG|png|BMP|bmp|doc|docx|pdf|xls|xlsx)$"
+var regImgStr = "\\.(JPG|jpg|jpeg|JPEG|GIF|gif|PNG|png|BMP|bmp|doc|docx|pdf|xls|xlsx)$"
 var regImg *regexp.Regexp
 
 func init() {
@@ -145,6 +145,61 @@ func DownloadAdv(retLen *int64, downloaderid, url, method string, reqparam, head
 	}
 }
 
+func DownloadAdvNew(downloaderid, url, method string, reqparam, head map[string]interface{}, mycookie []*http.Cookie, encoding string, useproxy, ishttps bool, code string, timeout int64) (string, []*http.Cookie, map[string]interface{}) {
+	defer mu.Catch()
+	msgid := mu.UUID(8)
+	if len(head) < 1 {
+		l := len(agent.UserAgents["common"])
+		r := rand.New(rand.NewSource(time.Now().UnixNano()))
+		head["User-Agent"] = agent.UserAgents["common"][r.Intn(l)]
+	}
+	var ret []byte
+	var err error
+	if downloaderid == "" {
+		ret, err = Msclient.Call("", msgid, mu.SERVICE_DOWNLOAD, mu.SENDTO_TYPE_RAND_RECIVER, map[string]interface{}{
+			"url":      url,
+			"method":   method,
+			"head":     head,
+			"reqparam": reqparam,
+			"cookie":   mycookie,
+			"encoding": encoding,
+			"useproxy": useproxy,
+			"ishttps":  ishttps,
+			"new":      true,
+		}, timeout)
+	} else {
+		if isAvailable(downloaderid) {
+			ret, err = Msclient.Call(downloaderid, msgid, mu.SERVICE_DOWNLOAD, mu.SENDTO_TYPE_P2P, map[string]interface{}{
+				"url":      url,
+				"method":   method,
+				"head":     head,
+				"reqparam": reqparam,
+				"cookie":   mycookie,
+				"encoding": encoding,
+				"useproxy": useproxy,
+				"ishttps":  ishttps,
+				"new":      true,
+			}, timeout)
+		} else {
+			return "", nil, nil
+		}
+	}
+	if err != nil {
+		str := code + "方法DownloadAdv,url:" + url + ",err:" + err.Error()
+		logger.Error(str, timeout)
+	}
+	tmp := map[string]interface{}{}
+	json.Unmarshal(ret, &tmp)
+	cooks := lu.ParseHttpCookie(tmp["cookie"])
+	headers, _ := tmp["header"].(map[string]interface{})
+	if v, ok := tmp["code"].(string); ok && v == "200" {
+		bs, _ := base64.StdEncoding.DecodeString(tmp["content"].(string))
+		return string(bs), cooks, headers
+	} else {
+		return "", nil, nil
+	}
+}
+
 //下载附件
 func DownloadFile_bak(downloaderid, url, method string, reqparam, head map[string]interface{}, mycookie []*http.Cookie, encoding string, useproxy, ishttps bool, code string, timeout int64) []byte {
 	defer mu.Catch()

src/spider/script.go

@@ -230,6 +230,37 @@ func (s *Script) LoadScript(site, channel, user *string, code, script_file strin
 		s.LastThreeTimes = append(s.LastThreeTimes, end)
 		return 3
 	}))
+	s.L.SetGlobal("downloadAdvNew", s.L.NewFunction(func(S *lua.LState) int {
+		cookie := S.ToString(-1)
+		head := S.ToTable(-2)
+		param := S.ToTable(-3)
+		method := S.ToString(-4)
+		url := S.ToString(-5)
+		ishttps := S.ToBool(-6)
+		charset := S.ToString(-7)
+		if charset == "" {
+			charset = s.Encoding
+		}
+		var mycookie []*http.Cookie
+		json.Unmarshal([]byte(cookie), &mycookie)
+		var ret string
+		var retcookie []*http.Cookie
+		var headers = map[string]interface{}{}
+		if param == nil {
+			ptext := map[string]interface{}{"text": S.ToString(-3)}
+			ret, retcookie, headers = DownloadAdvNew(s.Downloader, url, method, ptext, util.GetTable(head), mycookie, charset, s.Userproxy, ishttps, s.SCode, s.Timeout)
+		} else {
+			ret, retcookie, headers = DownloadAdvNew(s.Downloader, url, method, util.GetTable(param), util.GetTable(head), mycookie, charset, s.Userproxy, ishttps, s.SCode, s.Timeout)
+		}
+		S.Push(lua.LString(ret))
+		scookie, _ := json.Marshal(retcookie)
+		S.Push(lua.LString(scookie))
+		hTable := util.MapToLuaTable(S, headers)
+		S.Push(hTable)
+		atomic.AddInt32(&s.ToDayRequestNum, 1)
+		atomic.AddInt32(&s.TotalRequestNum, 1)
+		return 3
+	}))
 	//下载附件downloadFile(url,method,param,head,cookie,fileName)
 	//s.L.SetGlobal("downloadFile", s.L.NewFunction(func(S *lua.LState) int {
 	//	if s.FileLastThreeTimes == nil {
@@ -830,10 +861,14 @@ func (s *Script) LoadScript(site, channel, user *string, code, script_file strin
 	}))
 	//支持替换
 	s.L.SetGlobal("replace", s.L.NewFunction(func(S *lua.LState) int {
+		n := S.ToInt(-4)
+		if n <= 0 {
+			n = -1
+		}
 		text := S.ToString(-3)
 		old := S.ToString(-2)
 		repl := S.ToString(-1)
-		text = strings.Replace(text, old, repl, -1)
+		text = strings.Replace(text, old, repl, n)
 		S.Push(lua.LString(text))
 		return 1
 	}))
@@ -1202,14 +1237,23 @@ func (s *Script) LoadScript(site, channel, user *string, code, script_file strin
 	s.L.SetGlobal("downloadByChrome", s.L.NewFunction(func(S *lua.LState) int {
 		timeout := S.ToInt64(-2)
 		taskStr := S.ToString(-1)
-		chromeActions := []util.ChromeActions{}
-		if json.Unmarshal([]byte(taskStr), &chromeActions) == nil {
-			chromeTask := util.ChromeTask{
-				TimeOut: timeout,
-				Actions: chromeActions,
+		cam := util.ChromeActionMap{}
+		if json.Unmarshal([]byte(taskStr), &cam) == nil {
+			if len(cam.BaseActions) > 0 {
+				if len(cam.RangeActions) > 0 && cam.RangeTimes > 0 {
+					for times := 1; times <= cam.RangeTimes; times++ {
+						cam.BaseActions = append(cam.BaseActions, cam.RangeActions...)
+					}
+				}
+				chromeTask := util.ChromeTask{
+					TimeOut: timeout,
+					Actions: cam.BaseActions,
+				}
+				ret := DownloadByChrome(s.SCode, s.Downloader, chromeTask, s.Timeout)
+				S.Push(util.MapToTable(S, ret))
+			} else {
+				S.Push(S.NewTable())
 			}
-			ret := DownloadByChrome(s.SCode, s.Downloader, chromeTask, s.Timeout)
-			S.Push(util.MapToTable(S, ret))
 		} else {
 			S.Push(S.NewTable())
 		}