chromedp操作相关优化；download下载方法优化；

src/front/spider.go

@@ -56,7 +56,10 @@ type Step2 struct {
 	DateFormat     string
 	Expert         string
 	Types          int
-	Chrome         string
+	//chromedp相关
+	Chrome      string
+	RangeChrome string
+	RangeTimes  int
 }
 
 type Step3 struct {
@@ -350,8 +353,9 @@ func (f *Front) SaveStep() {
 				f.Step3.ContentChooser,
 				f.Step3.ElementChooser,
 			}
-			param_list_chrome, param_content_chrome := []sp.ChromeActions{}, []sp.ChromeActions{}
+			param_list_chrome, param_list_rangechrome, param_content_chrome := []sp.ChromeActions{}, []sp.ChromeActions{}, []sp.ChromeActions{}
 			json.Unmarshal([]byte(f.Step2.Chrome), &param_list_chrome)
+			json.Unmarshal([]byte(f.Step2.RangeChrome), &param_list_rangechrome)
 			json.Unmarshal([]byte(f.Step3.Chrome), &param_content_chrome)
 			param := map[string]interface{}{}
 			common[4] = 1
@@ -362,6 +366,8 @@ func (f *Front) SaveStep() {
 			param["param_time"] = ptime
 			param["param_list"] = list
 			param["param_list_chrome"] = param_list_chrome
+			param["param_list_rangechrome"] = param_list_rangechrome
+			param["param_list_rangetimes"] = f.Step2.RangeTimes
 			param["param_content_chrome"] = param_content_chrome
 			param["param_content"] = content
 			param["type_time"] = f.Step1.Types
@@ -372,6 +378,7 @@ func (f *Front) SaveStep() {
 			param["str_time"] = f.Step1.Expert
 			param["str_list"] = f.Step2.Expert
 			param["str_list_chrome"] = f.Step2.Chrome
+			param["str_list_rangechrome"] = f.Step2.RangeChrome
 			param["str_content_chrome"] = f.Step3.Chrome
 			param["str_content"] = f.Step3.Expert
 			param["comeintime"] = comeintime
@@ -440,7 +447,7 @@ func (f *Front) SaveStep() {
 				param["spidermovevent"] = movevent
 			}
 			//开发人员修改爬虫节点后，在审核人员上架时，要在原来的节点下架，临时记录要下架的节点downevent
-			if event := qu.IntAll((*one)["event"]); event != tmpEvent && event != 7000 {
+			if event := qu.IntAll((*one)["event"]); event != tmpEvent && (*one)["downevent"] == nil {
 				param["downevent"] = event
 			}
 			//三级页复制
@@ -448,7 +455,7 @@ func (f *Front) SaveStep() {
 			param["iscopycontent"] = f.StepRe3.Checked
 			//
 			param["listisfilter"] = ListFilterReg.MatchString(f.Step2.Expert) //列表页校验是否含“--关键词过滤”
-			checkLua := LuaTextCheck(infoformat, param, param_list_chrome, param_content_chrome, msgResult)
+			checkLua := LuaTextCheck(infoformat, param, param_list_chrome, param_list_rangechrome, param_content_chrome, msgResult)
 			//checkLua := LuaTextCheck(f.Base.SpiderCode, f.Step2.Expert, f.Step3.Expert, f.Step2.Types, infoformat, model, msgResult)
 			if !checkLua {
 				issave := spider.SaveSpider(code, param) //保存脚本
@@ -633,7 +640,7 @@ func LuaTextCheck(code, list, detail string, type_list, infoformat int, model ma
 	msgResult["err"] = errmsg
 	return errmsg != ""
 }*/
-func LuaTextCheck(infoformat int, param map[string]interface{}, param_list_chrome, param_content_chrome []sp.ChromeActions, msgResult map[string]string) bool {
+func LuaTextCheck(infoformat int, param map[string]interface{}, param_list_chrome, param_list_rangechrome, param_content_chrome []sp.ChromeActions, msgResult map[string]string) bool {
 	code := qu.ObjToString(param["code"])
 	list := qu.ObjToString(param["str_list"])
 	detail := qu.ObjToString(param["str_content"])
@@ -655,7 +662,14 @@ func LuaTextCheck(infoformat int, param map[string]interface{}, param_list_chrom
 		if strings.Contains(list, "downloadByChrome") { //chrome下载方法动作参数判断
 			for _, act := range param_list_chrome {
 				if act.Action != "changeip" && act.Param == "" {
-					errmsg += "列表页chrome模式'" + act.Action + "'动作未填写参数，填写后注意Ctrl+F10重新插入代码；"
+					errmsg += "列表页chrome模式'" + act.Action + "'基础动作未填写参数，填写后注意Ctrl+F10重新插入代码；"
+					break
+				}
+			}
+			for _, act := range param_list_rangechrome {
+				if act.Action != "changeip" && act.Param == "" {
+					errmsg += "列表页chrome模式'" + act.Action + "'循环动作未填写参数，填写后注意Ctrl+F10重新插入代码；"
+					break
 				}
 			}
 		}
@@ -768,18 +782,43 @@ func (f *Front) RunStep() {
 			}
 		case "Step2": //list
 			if imodal == 2 { //chromedp测试
-				chromeActions := []sp.ChromeActions{}
-				var result []interface{}
-				if json.Unmarshal([]byte(f.Step2.Chrome), &chromeActions) == nil {
+				//基础动作
+				if f.Step2.Chrome != "" {
+					chromeActions := []sp.ChromeActions{}
+					rangeChromeActions := []sp.ChromeActions{}
+					var result []interface{}
+					//格式化基础动作
+					if json.Unmarshal([]byte(f.Step2.Chrome), &chromeActions) != nil {
+						f.ServeJson("chrome基础动作格式化错误")
+						return
+					}
+					//格式化循环动作
+					if f.Step2.RangeChrome != "" {
+						if json.Unmarshal([]byte(f.Step2.RangeChrome), &rangeChromeActions) != nil {
+							f.ServeJson("chrome循环动作格式化错误")
+							return
+						}
+					}
+					//拼装基础和循环动作
+					if len(rangeChromeActions) > 0 && f.Step2.RangeTimes > 0 {
+						for times := 1; times <= f.Step2.RangeTimes; times++ {
+							chromeActions = append(chromeActions, rangeChromeActions...)
+						}
+					}
 					chromeTask := sp.ChromeTask{
 						TimeOut: 120, //测试默认两分钟
 						Actions: chromeActions,
 					}
 					result = spider.DownloadByChrome(downloadnode, "", chromeTask, 150)
+					resultMap := map[int]interface{}{}
+					for i, r := range result {
+						resultMap[i] = r
+					}
+					f.ServeJson(resultMap)
 				} else {
-					result = append(result, "chrome task json 格式化错误")
+					f.ServeJson("未进行动作编排")
 				}
-				f.ServeJson(result)
+
 			} else {
 				addrs := strings.Split(f.Step2.Listadds, "\n")
 				if len(addrs) > 0 {
@@ -834,7 +873,11 @@ func (f *Front) RunStep() {
 				} else {
 					result = append(result, "chrome task json 格式化错误")
 				}
-				f.ServeJson(result)
+				resultMap := map[int]interface{}{}
+				for i, r := range result {
+					resultMap[i] = r
+				}
+				f.ServeJson(resultMap)
 			} else {
 				content := []interface{}{
 					f.Step3.ContentChooser,
@@ -1405,8 +1448,9 @@ func UpStateAndUpSpider(code, id, reason, username string, state int) (bool, err
 				if upresult && err == nil {
 					unset = map[string]interface{}{"downevent": ""}
 				}
+			} else {
+				upresult, err = spider.UpdateSpiderByCodeState(code, "6", event)
 			}
-			upresult, err = spider.UpdateSpiderByCodeState(code, "6", event)
 			qu.Debug("下架:", upresult, code)
 			if upresult && err == nil {
 				upresult, err = spider.UpdateSpiderByCodeState(code, fmt.Sprint(state), event)

src/spider/download.go

@@ -19,7 +19,7 @@ import (
 	"github.com/surfer/agent"
 )
 
-var regImgStr = "\\.(JPG|jpg|GIF|gif|PNG|png|BMP|bmp)$"
+var regImgStr = "\\.(JPG|jpg|jpeg|JPEG|GIF|gif|PNG|png|BMP|bmp|doc|docx|pdf|xls|xlsx)$"
 var regImg *regexp.Regexp
 var GarbledCodeReg = regexp.MustCompile("[纰锟绲庯卞鍤滐銇鐟閫嚜鎯壐璩鏉彲鍋撅绺閲嗭絣鐤鏅盫鎽亰寰钂鎳鍒鐏宀婾嗚亗鎬憰攬鍙嶁鑻疐璁鐞鏇顭庮渾寮鑶剸鐙鈪鍐実綍擄鐒鐛绫瀵珐鍡閬栬憟灞綅顡韪忚鍓笉犵鍎鐥慪璜钀氭畯焛鎲顏熺崿鍜鍩僜鍚褰囶鍘櫥闀撹棢檅閯嗏絖灦戝閹涜闇鐮捒鈥璺籏绶澶鎷樺鍌絒嗘鍊ク鐧榦璞嚟鍢鐡瓼屾煢宄鑽畵鎭鈹鑷稛磭鏋孊钄狅絆鐘塋尟鑺絍绂绗嘐幇璨閾戭嚦鐫婅檴碭妤鑴厷挰鐜縒闆憁鏃鐗猒鏁橈顤秨哵鍧紛濊閷顥閺惪鐓嶈亙濠掗帾媞鏀慿瓙鎺闁鎰鑸鎹皝鍔鍦骞閶鍞挾鎴竗閵繉闋戞籅閽欏閼縲鐣呮墔顐ら憼檾锝挻顚炶姂剾鐑鐭潛閰涳楂懘願澧亣倴鐦忕嫄刡灏棙宓媐铇甀鏂楁従態瀹揕闃姒炲矕鏌眱鍍熸腹儝绱獻鐬鑵矦鍝嗗墹崇琛勭仈濴顒剭閴鍏鐝曨锛よ顧勯槈夊潏鐖垚矑鍛瞋終缂鐪鍠鏆妫攏顪娌濆嘇璎厫鍗閮顝給榇婂唭姘燁鏍鑹笎爑嚔槌瀣糵炵櫤鐎闅ゅ類鐨夛绋搕缃娉犲搻鐠儧鋸闉攜楸ㄨ埧欒闊垱鈩厔弐顠拵鑾]+")
 
@@ -173,7 +173,6 @@ func DownloadAdvNew(downloadnode, downloaderid, url, method string, reqparam, he
 		r := rand.New(rand.NewSource(time.Now().UnixNano()))
 		head["User-Agent"] = agent.UserAgents["common"][r.Intn(l)]
 	}
-	isImg := regImg.MatchString(url)
 	var ret []byte
 	var err error
 	if downloaderid == "" {
@@ -213,13 +212,8 @@ func DownloadAdvNew(downloadnode, downloaderid, url, method string, reqparam, he
 	cooks := util.ParseHttpCookie(tmp["cookie"])
 	headers, _ := tmp["header"].(map[string]interface{})
 	if v, ok := tmp["code"].(string); ok && v == "200" {
-		if isImg {
-			bs, _ := tmp["content"].(string)
-			return string(bs), cooks, headers
-		} else {
-			bs, _ := base64.StdEncoding.DecodeString(tmp["content"].(string))
-			return string(bs), cooks, headers
-		}
+		bs, _ := base64.StdEncoding.DecodeString(tmp["content"].(string))
+		return string(bs), cooks, headers
 	} else {
 		return "", nil, nil
 	}

src/spider/script.go

@@ -424,10 +424,14 @@ func (s *Script) LoadScript(site *string, downloadnode, script string, isfile ..
 	}))
 	//支持替换
 	s.L.SetGlobal("replace", s.L.NewFunction(func(S *lua.LState) int {
+		n := S.ToInt(-4)
+		if n <= 0 {
+			n = -1
+		}
 		text := S.ToString(-3)
 		old := S.ToString(-2)
 		repl := S.ToString(-1)
-		text = strings.Replace(text, old, repl, -1)
+		text = strings.Replace(text, old, repl, n)
 		S.Push(lua.LString(text))
 		return 1
 	}))
@@ -884,14 +888,23 @@ func (s *Script) LoadScript(site *string, downloadnode, script string, isfile ..
 	s.L.SetGlobal("downloadByChrome", s.L.NewFunction(func(S *lua.LState) int {
 		timeout := S.ToInt64(-2)
 		taskStr := S.ToString(-1)
-		chromeActions := []sp.ChromeActions{}
-		if json.Unmarshal([]byte(taskStr), &chromeActions) == nil {
-			chromeTask := sp.ChromeTask{
-				TimeOut: timeout,
-				Actions: chromeActions,
+		cam := sp.ChromeActionMap{}
+		if json.Unmarshal([]byte(taskStr), &cam) == nil {
+			if len(cam.BaseActions) > 0 {
+				if len(cam.RangeActions) > 0 && cam.RangeTimes > 0 {
+					for times := 1; times <= cam.RangeTimes; times++ {
+						cam.BaseActions = append(cam.BaseActions, cam.RangeActions...)
+					}
+				}
+				chromeTask := sp.ChromeTask{
+					TimeOut: timeout,
+					Actions: cam.BaseActions,
+				}
+				ret := DownloadByChrome(downloadnode, s.Downloader, chromeTask, s.Timeout)
+				S.Push(sp.MapToTable(S, ret))
+			} else {
+				S.Push(lua.LString("未设置基础循环动作"))
 			}
-			ret := DownloadByChrome(downloadnode, s.Downloader, chromeTask, s.Timeout)
-			S.Push(sp.MapToTable(S, ret))
 		} else {
 			S.Push(lua.LString("chrome task json 格式化错误"))
 		}

src/timetask/timetask.go

@@ -208,7 +208,14 @@ func SpiderMoveEvent() {
 				//	set["str_content"] = str_recontent
 				//	set["str_recontent"] = str_content
 				//}
-				upOk := util.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": set}, false, false)
+				upset := map[string]interface{}{
+					"$set": set,
+				}
+				if downevent := qu.IntAll((*lua)["downevent"]); downevent != 0 {
+					spider.UpdateSpiderByCodeState(code, "6", downevent)
+					upset["$unset"] = map[string]interface{}{"downevent": ""}
+				}
+				upOk := util.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, upset, false, false)
 				qu.Debug(code, "脚本更新成功：", upOk)
 				if upOk {
 					upresult, err = spider.UpdateSpiderByCodeState(code, "5", newevent) //脚本上架

src/web/staticres/js/common.js

@@ -54,6 +54,7 @@ common.form.isCheck=function(id,func){
 common.form.serializeArray=function(id){
 	var serializeArray={};
 	var chromeArr = [];
+	var rangeChromeArr = [];
 	$("#"+id+" [name]").each(function(){
 		var name=$(this).prop("name");
 		var value=$(this).prop("value");
@@ -72,7 +73,16 @@ common.form.serializeArray=function(id){
 			value=value?value:"";
 		}
 		//组合有序的chrome
-		if(name.includes("Chrome")){
+		if(name.includes("Range.Chrome")){//循环动作
+			var rangeChromeMap = {};
+			rangeChromeMap["action"] = name.replace("Range.Chrome.","");
+			rangeChromeMap["param"] = value
+			var selector_val = $(this).parents(".chrome-actions").find("[sel='"+name+".selector']").val();
+			if (selector_val) {
+				rangeChromeMap["selector"]= selector_val
+			}
+			rangeChromeArr.push(rangeChromeMap);
+		}else if (name.includes("Chrome")){//基础动作
 			var chromeMap = {};
 			chromeMap["action"] = name.replace("Chrome.","");
 			chromeMap["param"] = value
@@ -88,6 +98,9 @@ common.form.serializeArray=function(id){
 	if (chromeArr.length > 0){
 		serializeArray[id+".Chrome"] = JSON.stringify(chromeArr);
 	}
+	if (rangeChromeArr.length > 0){
+		serializeArray[id+".RangeChrome"] = JSON.stringify(rangeChromeArr);
+	}
 	return serializeArray
 }
 common.form.submit=function(){}
@@ -162,25 +175,61 @@ common.spider.initMirror=function()	{
 					window.alert("非chrome爬虫，禁止使用chrome下载方法");
 					return
 				}
+				var regex = /{([^}]*)}/g;
 				var str_tab = "\t".repeat(cm.getCursor().ch);
 				var data = common.form.serializeArray("Step2");
-				var chromeActionsStr = data["Step2.Chrome"];
-				var luaText = `local timeout = 60\n${str_tab}local chromeTask = [[\n${str_tab}\t[\n${str_tab}\t\t`;
-				if (chromeActionsStr){
-					var chromeActionMapStr = chromeActionsStr.substring(1,chromeActionsStr.length - 1);
-					var regex = /{([^}]*)}/g;
-					var chromeActionsStrArr = chromeActionMapStr.match(regex);
-					for (var i in chromeActionsStrArr){
-						if (i == chromeActionsStrArr.length -1){
-							luaText += chromeActionsStrArr[i]+`\n${str_tab}\t`;
+				var baseChromeActionsStr = data["Step2.Chrome"];
+				var luaText = `local timeout = 60\n${str_tab}local chromeTask = [[\n${str_tab}\t{\n${str_tab}\t\t`;
+				var baseText = ``;
+				var rangText = ``;
+				var timesText = ``;
+				if (baseChromeActionsStr){
+					var rangeChromeActionsStr = data["Step2.RangeChrome"];
+					var rangeTimes = data["Step2.RangeTimes"];
+					if(rangeChromeActionsStr) {
+						if(!rangeTimes){
+							window.alert("未设置循环动作执行次数");
+							return;
+						}else{
+							timesText = `${str_tab}"range-times":`+rangeTimes+`,\n${str_tab}${str_tab}`
+						}
+						//循环动作
+						var rangeChromeActionMapStr = rangeChromeActionsStr.substring(1,rangeChromeActionsStr.length - 1);
+						var rangeChromeActionsStrArr = rangeChromeActionMapStr.match(regex);
+						for (var i in rangeChromeActionsStrArr){
+							if (i == rangeChromeActionsStrArr.length -1){
+								 rangText += rangeChromeActionsStrArr[i]+`\n${str_tab}\t`;
+							}else{
+								rangText += rangeChromeActionsStrArr[i]+`,\n${str_tab}\t\t\t`;
+							}
+						}
+						rangText = `"range-actions":[\n${str_tab}${str_tab}\t`+rangText+`\t]\n${str_tab}`
+					}
+					//基础动作
+					var baseChromeActionMapStr = baseChromeActionsStr.substring(1,baseChromeActionsStr.length - 1);
+					var baseChromeActionsStrArr = baseChromeActionMapStr.match(regex);
+					for (var i in baseChromeActionsStrArr){
+						var baseActionText = baseChromeActionsStrArr[i];
+						if (baseActionText.includes("navigate")){
+							baseActionText = `{"action":"navigate","param":"]]..href..[["}`
+						}
+						if (i == baseChromeActionsStrArr.length -1){
+							baseText += baseActionText+`\n${str_tab}\t`;
 						}else{
-							luaText += chromeActionsStrArr[i]+`,\n${str_tab}\t\t`;
+							baseText += baseActionText+`,\n${str_tab}\t\t\t`;
 						}
 					}
+					if(rangText){
+						baseText = `"base-actions":[\n${str_tab}${str_tab}\t`+baseText+`\t],\n${str_tab}`
+					}else{
+						baseText = `"base-actions":[\n${str_tab}${str_tab}\t`+baseText+`\t]\n${str_tab}`
+					}
 				}else{
-					luaText += `添加执行动作\n${str_tab}\t`;
+					//luaText += `添加基础执行动作\n${str_tab}\t`;
+					window.alert("未设置基础循环动作");
+					return;
 				}
-				luaText += `]\n${str_tab}]]\n${str_tab}local contenthtml = downloadByChrome(timeout,chromeTask)`
+				luaText += baseText + timesText + rangText + `\t}\n${str_tab}]]\n${str_tab}local contenthtmltable = downloadByChrome(timeout,chromeTask)`
 				cm.replaceSelection(luaText);
 			},
 		    "Esc": function(cm) {
@@ -218,26 +267,63 @@ common.spider.initMirror=function()	{
 					window.alert("非chrome爬虫，禁止使用chrome下载方法");
 					return
 				}
+				var regex = /{([^}]*)}/g;
 				var str_tab = "\t".repeat(cm.getCursor().ch);
 				var data = common.form.serializeArray("Step3");
-				var chromeActionsStr = data["Step3.Chrome"];
-				var luaText = `local timeout = 60\n${str_tab}local chromeTask = [[\n${str_tab}\t[\n${str_tab}\t\t`;
-				if (chromeActionsStr){
-					var chromeActionMapStr = chromeActionsStr.substring(1,chromeActionsStr.length - 1);
-					var regex = /{([^}]*)}/g;
-					var chromeActionsStrArr = chromeActionMapStr.match(regex);
-					for (var i in chromeActionsStrArr){
-						if (i == chromeActionsStrArr.length -1){
-							luaText += chromeActionsStrArr[i]+`\n${str_tab}\t`;
+				var baseChromeActionsStr = data["Step3.Chrome"];
+				var luaText = `local timeout = 60\n${str_tab}local chromeTask = [[\n${str_tab}\t{\n${str_tab}\t\t`;
+				var baseText = ``;
+				var rangText = ``;
+				var timesText = ``;
+				if (baseChromeActionsStr){
+					var rangeChromeActionsStr = data["Step3.RangeChrome"];
+					var rangeTimes = data["Step3.RangeTimes"];
+					if(rangeChromeActionsStr) {
+						if(!rangeTimes){
+							window.alert("未设置循环动作执行次数");
+							return;
+						}else{
+							timesText = `${str_tab}"range-times":`+rangeTimes+`,\n${str_tab}${str_tab}`
+						}
+						//循环动作
+						var rangeChromeActionMapStr = rangeChromeActionsStr.substring(1,rangeChromeActionsStr.length - 1);
+						var rangeChromeActionsStrArr = rangeChromeActionMapStr.match(regex);
+						for (var i in rangeChromeActionsStrArr){
+							if (i == rangeChromeActionsStrArr.length -1){
+								rangText += rangeChromeActionsStrArr[i]+`\n${str_tab}\t`;
+							}else{
+								rangText += rangeChromeActionsStrArr[i]+`,\n${str_tab}\t\t\t`;
+							}
+						}
+						rangText = `"range-actions":[\n${str_tab}${str_tab}\t`+rangText+`\t]\n${str_tab}`
+					}
+					//基础动作
+					var baseChromeActionMapStr = baseChromeActionsStr.substring(1,baseChromeActionsStr.length - 1);
+					var baseChromeActionsStrArr = baseChromeActionMapStr.match(regex);
+					for (var i in baseChromeActionsStrArr){
+						var baseActionText = baseChromeActionsStrArr[i];
+						if (baseActionText.includes("navigate")){
+							baseActionText = `{"action":"navigate","param":"]]..data["href"]..[["}`
+						}
+						if (i == baseChromeActionsStrArr.length -1){
+							baseText += baseActionText+`\n${str_tab}\t`;
 						}else{
-							luaText += chromeActionsStrArr[i]+`,\n${str_tab}\t\t`;
+							baseText += baseActionText+`,\n${str_tab}\t\t\t`;
 						}
 					}
+					if(rangText){
+						baseText = `"base-actions":[\n${str_tab}${str_tab}\t`+baseText+`\t],\n${str_tab}`
+					}else{
+						baseText = `"base-actions":[\n${str_tab}${str_tab}\t`+baseText+`\t]\n${str_tab}`
+					}
 				}else{
-					luaText += `添加执行动作\n${str_tab}\t`;
+					//luaText += `添加基础执行动作\n${str_tab}\t`;
+					window.alert("未设置基础循环动作");
+					return;
 				}
-				luaText += `]\n${str_tab}]]\n${str_tab}local contenthtml = downloadByChrome(timeout,chromeTask)`
+				luaText += baseText + timesText + rangText + `\t}\n${str_tab}]]\n${str_tab}local contenthtmltable = downloadByChrome(timeout,chromeTask)`
 				cm.replaceSelection(luaText);
+
 			},
 		    "Esc": function(cm) {
 		        if (cm.getOption("fullScreen")) cm.setOption("fullScreen", false);

src/web/templates/index.html

@@ -321,7 +321,6 @@ $(function(){
 			  }},
 			//更新平台
 			{"targets":[10], createdCell: function (cell, cellData, rowData, rowIndex, colIndex) {
-				// {{if gt (session "auth") 2}}
 				  var aInput;
 				  $(cell).click(function () {
 					$(this).html(createComboxPlatform(rowData.code,rowData._id));
@@ -339,7 +338,6 @@ $(function(){
 					  ttable.cell(cell).data(text);
 					  cellData = text;
 				  });
-				// {{end}}
 			  }},
 			//爬虫挂起
 			{"targets":[11], createdCell: function (cell, cellData, rowData, rowIndex, colIndex) {
@@ -740,6 +738,33 @@ $(function(){
 			}
 		})
 	}
+function createComboxPlatform(code,id){
+	return "<select onchange='updateesp(this.value,\"platform\",\"\",\""+code+"\",\""+id+"\")' class='form-control input-sm'>"+
+			"<option value='golua平台'>golua平台</option>"+
+			"<option value='chrome'>chrome</option>"+
+			"<option value='python'>python</option>"+
+			"<option value='通用爬虫'>通用爬虫</option>"+
+			"</select>"
+};
+function updateesp(val,w,s,c,id){
+	$.ajax({
+		url:"/center/spider/updateesp",
+		type:"post",
+		data:{
+			"val":val,
+			"w":w,
+			"id":id,
+			"s":s,
+			"c":c
+		},
+		success:function(r){
+			if(r=="n"){
+				showTip("修改失败", 1000, function() {});
+			}
+			ttable.ajax.reload();
+		}
+	})
+}
 {{if gt (session "auth") 1}}
 	function refuse(me){
 		var $r=window.prompt("确定打回？请输入打回原因")
@@ -958,33 +983,6 @@ $(function(){
             "<option disabled value='10'>已上线</option>"+
           +"</select>"
   };
-    function createComboxPlatform(code,id){
-    	return "<select onchange='updateesp(this.value,\"platform\",\"\",\""+code+"\",\""+id+"\")' class='form-control input-sm'>"+
-    			  "<option value='golua平台'>golua平台</option>"+
-				"<option value='chrome'>chrome</option>"+
-				"<option value='python'>python</option>"+
-    				"<option value='通用爬虫'>通用爬虫</option>"+
-          "</select>"
-  };
-    function updateesp(val,w,s,c,id){
-    	$.ajax({
-			url:"/center/spider/updateesp",
-			type:"post",
-			data:{
-				"val":val,
-				"w":w,
-				"id":id,
-				"s":s,
-				"c":c
-			},
-		    success:function(r){
-			   if(r=="n"){
-				  showTip("修改失败", 1000, function() {});
-				}
-        		ttable.ajax.reload();
-			}
-		})
-  }
 {{end}}
 function spidercopy_cancel(){
 	$("#modal-spidercopy").modal("hide");

src/web/templates/spiderbase.html

@@ -14,6 +14,9 @@
 		margin-top: 7px;
 		color: red;
 	}
+	.chrome-task-block{
+		border-bottom: 1px solid #e5e5e5;
+	}
 </style>
 <div class="modal fade" id="modal-js">
 	<div class="modal-dialog">
@@ -536,7 +539,7 @@
 	</div>
 </form>
 <script>
-	var actionHtmlMap ={
+	var baseActionHtmlMap ={
 		"changeip":`
 			<div class="form-group chrome-actions">
 				<label class="control-label col-md-2">切换IP：</label>
@@ -658,7 +661,129 @@
 				<h6 style="color: red">（同级：0；父级：1；爷级：2）</h6>
 			</div>`,
 	}
-  function spiderTypeChange(){
+	var rangeActionHtmlMap ={
+		"changeip":`
+			<div class="form-group chrome-actions">
+				<label class="control-label col-md-2">切换IP：</label>
+				<i class="fa fa-plus-square" style="cursor: pointer" onclick="addChromeAction(this,1)"></i>&nbsp;
+				<i class="fa fa-minus-square" style="cursor: pointer" onclick="removeChromeAction(this)"></i>
+				<div class="col-md-5">
+					<input type="text" disabled class="form-control" name="Range.Chrome.changeip"/>
+				</div>
+			</div>`,
+		"navigate":`
+			<div class="form-group chrome-actions">
+				<label class="control-label col-md-2 ">列表页地址：</label>
+				<i class="fa fa-plus-square" onclick="addChromeAction(this,1)"></i>&nbsp;
+				<i class="fa fa-minus-square" style="cursor: pointer" onclick="removeChromeAction(this)"></i>
+<!--				<i class="fa fa-times-circle" style="color: red;margin-top: 7px;"></i>-->
+				<div class="col-md-5">
+					<input type="text" class="form-control" placeholder="请输入地址" name="Range.Chrome.navigate" />
+				</div>
+			</div>`,
+		"click":`
+			<div class="form-group chrome-actions">
+				<label class="control-label col-md-2">点击：</label>
+				<i class="fa fa-plus-square" style="cursor: pointer" onclick="addChromeAction(this,1)"></i>&nbsp;
+				<i class="fa fa-minus-square" style="cursor: pointer" onclick="removeChromeAction(this)"></i>
+				<div class="col-md-5">
+					<input type="text" class="form-control" placeholder="请输入selector" name="Range.Chrome.click" />
+				</div>
+				<label class="control-label col-md-2">Selector：</label>
+				<div class="col-md-2">
+					<select sel="Range.Chrome.click.selector">
+						<option>ByQuery</option>
+						<option>ByID</option>
+					</select>
+				</div>
+			</div>`,
+		"waitready":`
+			<div class="form-group chrome-actions">
+				<label class="control-label col-md-2">元素加载完毕：</label>
+				<i class="fa fa-plus-square" style="cursor: pointer" onclick="addChromeAction(this,1)"></i>&nbsp;
+				<i class="fa fa-minus-square" style="cursor: pointer" onclick="removeChromeAction(this)"></i>
+				<div class="col-md-5">
+					<input type="text" class="form-control" placeholder="请输入selector" name="Range.Chrome.waitready" />
+				</div>
+				<label class="control-label col-md-2">Selector：</label>
+				<div class="col-md-2">
+					<select sel="Range.Chrome.waitready.selector">
+						<option>ByQuery</option>
+						<option>ByID</option>
+					</select>
+				</div>
+			</div>`,
+		"waitvisible":`
+			<div class="form-group chrome-actions">
+				<label class="control-label col-md-2">元素可见：</label>
+				<i class="fa fa-plus-square" style="cursor: pointer" onclick="addChromeAction(this,1)"></i>&nbsp;
+				<i class="fa fa-minus-square" style="cursor: pointer" onclick="removeChromeAction(this)"></i>
+				<div class="col-md-5">
+					<input type="text" class="form-control" placeholder="请输入selector" name="Range.Chrome.waitvisible" />
+				</div>
+				<label class="control-label col-md-2">Selector：</label>
+				<div class="col-md-2">
+					<select sel="Range.Chrome.waitvisible.selector">
+						<option>ByQuery</option>
+						<option>ByID</option>
+					</select>
+				</div>
+			</div>`,
+		"wait":`
+			<div class="form-group chrome-actions">
+				<label class="control-label col-md-2">等待休息：</label>
+				<i class="fa fa-plus-square" style="cursor: pointer" onclick="addChromeAction(this,1)"></i>&nbsp;
+				<i class="fa fa-minus-square" style="cursor: pointer" onclick="removeChromeAction(this)"></i>
+				<div class="col-md-5">
+					<input type="text" class="form-control" placeholder="请输入等待时长（秒）" name="Range.Chrome.wait" />
+				</div>
+			</div>`,
+		"evaluate":`
+			<div class="form-group chrome-actions">
+				<label class="control-label col-md-2">执行JS：</label>
+				<i class="fa fa-plus-square" style="cursor: pointer" onclick="addChromeAction(this,1)"></i>&nbsp;
+				<i class="fa fa-minus-square" style="cursor: pointer" onclick="removeChromeAction(this)"></i>
+				<div class="col-md-5">
+					<input type="text" class="form-control" placeholder="请输入JS" name="Range.Chrome.evaluate" />
+				</div>
+			</div>`,
+		"outerhtml":`
+			<div class="form-group chrome-actions">
+				<label class="control-label col-md-2">获取HTML：</label>
+				<i class="fa fa-plus-square" style="cursor: pointer" onclick="addChromeAction(this,1)"></i>&nbsp;
+				<i class="fa fa-minus-square" style="cursor: pointer" onclick="removeChromeAction(this)"></i>
+				<div class="col-md-5">
+					<input type="text" class="form-control" placeholder="请输入selector" name="Range.Chrome.outerhtml" />
+				</div>
+				<label class="control-label col-md-2">Selector：</label>
+				<div class="col-md-2">
+					<select sel="Range.Chrome.outerhtml.selector">
+						<option>ByQuery</option>
+						<option>ByID</option>
+					</select>
+				</div>
+			</div>`,
+		"listhref":`
+			<div class="form-group chrome-actions">
+				<label class="control-label col-md-2">匹配列表链接：</label>
+				<i class="fa fa-plus-square" style="cursor: pointer" onclick="addChromeAction(this,1)"></i>&nbsp;
+				<i class="fa fa-minus-square" style="cursor: pointer" onclick="removeChromeAction(this)"></i>
+				<div class="col-md-5">
+					<input type="text" class="form-control" placeholder="请输入获取列表页a链接selector" name="Range.Chrome.listhref" />
+				</div>
+			</div>`,
+		"listhtml":`
+			<div class="form-group chrome-actions">
+				<label class="control-label col-md-2">获取列表HTML：</label>
+				<i class="fa fa-plus-square" style="cursor: pointer" onclick="addChromeAction(this,1)"></i>&nbsp;
+				<i class="fa fa-minus-square" style="cursor: pointer" onclick="removeChromeAction(this)"></i>
+				<div class="col-md-5">
+					<input type="text" class="form-control" placeholder="请输入获取列表页html与a链接的层级关系" name="Range.Chrome.listhtml" />
+				</div>
+				<h6 style="color: red">（同级：0；父级：1；爷级：2）</h6>
+			</div>`,
+	}
+	function spiderTypeChange(){
     var val = $("#spiderType").val();
     if (val == "increment") {//选择增量模式
       $("#historymaxpage").addClass("hide");

src/web/templates/spideredit.html

@@ -361,6 +361,7 @@
 			common.spider.getModel(common.spider.setModel);
 
 		})
+		//爬虫测试
 		$("legend a").click(function(event,editor){
       		var node = $(this).parent().find("input[type='radio']:checked").val()//测试线路
 			var target=$(this).attr("for");
@@ -595,7 +596,14 @@
 	  })
   }
   //添加chrome动作
-  function addChromeAction(a){
+  function addChromeAction(a,actionType){
+	  // return [{name:'切换IP（changeip）',value:'changeip'},{name:'链接地址（navigate）',value:'navigate'},{name:'点击（click）',value:'click'},{name:'元素加载完毕（waitready）',value:'waitready'},{name:'元素可见（waitvisible）',value:'waitvisible'},{name:'等待休息（wait）',value:'wait'},{name:'执行JS（evaluate）',value:'evaluate'},{name:'获取HTML（outerhtml）',value:'outerhtml'},{name:'匹配列表链接（listhref）',value:'listhref'},{name:'获取列表HTML（listhtml）',value:'listhtml'}];
+	  var actionData = [{name:'切换IP（changeip）',value:'changeip'},{name:'链接地址（navigate）',value:'navigate'},{name:'点击（click）',value:'click'},{name:'元素加载完毕（waitready）',value:'waitready'},{name:'等待休息（wait）',value:'wait'},{name:'执行JS（evaluate）',value:'evaluate'},{name:'获取HTML（outerhtml）',value:'outerhtml'}];
+	  var actionMap = baseActionHtmlMap;
+	  if (actionType == 1){
+		  actionData = [{name:'切换IP（changeip）',value:'changeip'},{name:'点击（click）',value:'click'},{name:'元素加载完毕（waitready）',value:'waitready'},{name:'等待休息（wait）',value:'wait'},{name:'执行JS（evaluate）',value:'evaluate'},{name:'获取HTML（outerhtml）',value:'outerhtml'}];
+		  actionMap = rangeActionHtmlMap;
+	  }
 	  parentNode = $(a).parent();
 	  //选择动作https://maplemei.gitee.io/xm-select/#/basic/autoRow
 	  xmSelect.get("#selectList").reset;//重置
@@ -625,15 +633,14 @@
 			  strict: false,
 		  },
 		  data: function () {
-			  // return [{name:'切换IP（changeip）',value:'changeip'},{name:'链接地址（navigate）',value:'navigate'},{name:'点击（click）',value:'click'},{name:'元素加载完毕（waitready）',value:'waitready'},{name:'元素可见（waitvisible）',value:'waitvisible'},{name:'等待休息（wait）',value:'wait'},{name:'执行JS（evaluate）',value:'evaluate'},{name:'获取HTML（outerhtml）',value:'outerhtml'},{name:'匹配列表链接（listhref）',value:'listhref'},{name:'获取列表HTML（listhtml）',value:'listhtml'}];
-			  return [{name:'切换IP（changeip）',value:'changeip'},{name:'链接地址（navigate）',value:'navigate'},{name:'点击（click）',value:'click'},{name:'元素加载完毕（waitready）',value:'waitready'},{name:'元素可见（waitvisible）',value:'waitvisible'},{name:'等待休息（wait）',value:'wait'},{name:'执行JS（evaluate）',value:'evaluate'},{name:'获取HTML（outerhtml）',value:'outerhtml'}];
+			  return actionData;
 		  },
 		  on: function (data) {
 			  var actions = data.arr;//arr:  当前多选已选中的数据
 			  actionsHtml = "";//重置
 			  for(var i in actions){
 				  var action = actions[i].value;
-				  actionsHtml = actionsHtml +actionHtmlMap[action];
+				  actionsHtml = actionsHtml +actionMap[action];
 			  }
 			  // parentNode.after(actionsHtml);
 			  // //change, 此次选择变化的数据,数组

src/web/templates/spiderview.html

@@ -63,10 +63,10 @@
 	</div>
 	
 	<script>
+	var actionsHtml = ""
+	var parentNode
+	var luaIsChrome = {{.T.lua.platform}} == "chrome";
 	$(function(){
-		var actionsHtml = ""
-		var parentNode
-		var luaIsChrome = {{.T.lua.platform}} == "chrome";
 		editor2_js = CodeMirror.fromTextArea($("#Step2_js")[0], {
 			theme:"solarized light",
 			lineNumbers: true,
@@ -254,5 +254,73 @@
 				}
 			{{end}}
 		})
+	//添加chrome动作
+	function addChromeAction(a,actionType){
+		// return [{name:'切换IP（changeip）',value:'changeip'},{name:'链接地址（navigate）',value:'navigate'},{name:'点击（click）',value:'click'},{name:'元素加载完毕（waitready）',value:'waitready'},{name:'元素可见（waitvisible）',value:'waitvisible'},{name:'等待休息（wait）',value:'wait'},{name:'执行JS（evaluate）',value:'evaluate'},{name:'获取HTML（outerhtml）',value:'outerhtml'},{name:'匹配列表链接（listhref）',value:'listhref'},{name:'获取列表HTML（listhtml）',value:'listhtml'}];
+		var actionData = [{name:'切换IP（changeip）',value:'changeip'},{name:'链接地址（navigate）',value:'navigate'},{name:'点击（click）',value:'click'},{name:'元素加载完毕（waitready）',value:'waitready'},{name:'等待休息（wait）',value:'wait'},{name:'执行JS（evaluate）',value:'evaluate'},{name:'获取HTML（outerhtml）',value:'outerhtml'}];
+		var actionMap = baseActionHtmlMap;
+		if (actionType == 1){
+			actionData = [{name:'切换IP（changeip）',value:'changeip'},{name:'点击（click）',value:'click'},{name:'元素加载完毕（waitready）',value:'waitready'},{name:'等待休息（wait）',value:'wait'},{name:'执行JS（evaluate）',value:'evaluate'},{name:'获取HTML（outerhtml）',value:'outerhtml'}];
+			actionMap = rangeActionHtmlMap;
+		}
+		parentNode = $(a).parent();
+		//选择动作https://maplemei.gitee.io/xm-select/#/basic/autoRow
+		xmSelect.get("#selectList").reset;//重置
+		xmSelect.render({
+			el: '#selectList',
+			model: {
+				icon: 'hidden',
+			},
+			toolbar:{
+				show: true,
+				showIcon: false,
+			},
+			checkbox: true,
+			repeat: true,
+			autoRow: true,
+			filterable: true,
+			prop: {name: 'name', value: 'value',},
+			height: '400px',
+			tips: '请选择',
+			autoRow:true,
+			tree: {
+				show: true,
+				showFolderIcon: false,
+				showLine: true,
+				indent: 20,
+				expandedKeys: [-3],
+				strict: false,
+			},
+			data: function () {
+				return actionData;
+			},
+			on: function (data) {
+				var actions = data.arr;//arr:  当前多选已选中的数据
+				actionsHtml = "";//重置
+				for(var i in actions){
+					var action = actions[i].value;
+					actionsHtml = actionsHtml +actionMap[action];
+				}
+				// parentNode.after(actionsHtml);
+				// //change, 此次选择变化的数据,数组
+				// var change = data.change;
+				// //isAdd, 此次操作是新增还是删除
+				// var isAdd = data.isAdd;
+				// console.log(change,isAdd)
+			},
+		});
+		$("#chrome-task").modal("show");
+	}
+
+	//保存动作
+	function saveActions() {
+		// $("#chrome-base-action-list").append(actionsHtml)
+		parentNode.after(actionsHtml);
+		$("#chrome-task").modal("hide");
+	}
+	//删除动作
+	function removeChromeAction(a) {
+		$(a).parents(".chrome-actions").remove();
+	}
 	</script>
 {{include "bottom.html"}}

src/web/templates/step2.html

@@ -14,26 +14,39 @@
 					</div>
 				</div>
 			</legend>
-			<p1>编辑动作集</p1>
-			<div class="form-group chrome-actions" style="display: inline-block">
+			<div class="chrome-task-block">
+				<p1>基础动作</p1>
+				<div class="form-group chrome-base-actions chrome-actions" style="display: inline-block">
 					<i class="fa fa-plus-square" onclick="addChromeAction(this)"></i>
-<!--				<label class="control-label col-md-2 ">列表页地址：</label>-->
-<!--				<i class="fa fa-plus-square" onclick="addChromeAction(this)"></i>&nbsp;-->
-<!--				<i class="fa fa-times-circle" style="color: red;margin-top: 7px;"></i>-->
-<!--				<div class="col-md-5">-->
-<!--					<input type="text" class="form-control" placeholder="请输入地址" name="Chrome.navigate" />-->
-<!--				</div>-->
-<!--				<h6 style="color: red">（不可删除）</h6>-->
+					<!--				<label class="control-label col-md-2 ">列表页地址：</label>-->
+					<!--				<i class="fa fa-plus-square" onclick="addChromeAction(this)"></i>&nbsp;-->
+					<!--				<i class="fa fa-times-circle" style="color: red;margin-top: 7px;"></i>-->
+					<!--				<div class="col-md-5">-->
+					<!--					<input type="text" class="form-control" placeholder="请输入地址" name="Chrome.navigate" />-->
+					<!--				</div>-->
+					<!--				<h6 style="color: red">（不可删除）</h6>-->
+				</div>
+			</div>
+
+			<div class="chrome-task-block">
+				<p1>循环动作</p1>
+				<div class="form-group chrome-range-actions chrome-actions" style="display: inline-block">
+					<i class="fa fa-plus-square" onclick="addChromeAction(this,1)"></i>
+					<p1 style="margin-left: 22px">循环次数：</p1>
+					<input type="text" style="width: 35px;text-align: center;" class="form-inline" value="{{.T.lua.param_list_rangetimes}}" name="Step2.RangeTimes" />
+					<p1 style="color: red">（不能为0）</p1>
+				</div>
 			</div>
 			<script>
+				//基础动作
 				var Step2_Chrome = {{.T.lua.param_list_chrome}};
 				//先铺html
 				var Step2ChromeHtml = "";
 				for (var i in Step2_Chrome){
 					 var action = Step2_Chrome[i].action
-					 Step2ChromeHtml+=actionHtmlMap[action];
+					 Step2ChromeHtml+=baseActionHtmlMap[action];
 				}
-				$("#Step2 .chrome-actions").after(Step2ChromeHtml);
+				$("#Step2 .chrome-base-actions").after(Step2ChromeHtml);
 				//再赋值
 				for (var i in Step2_Chrome){
 					var action = Step2_Chrome[i].action
@@ -44,6 +57,25 @@
 						$("#Step2 [sel='Chrome."+action+".selector']").val(selector);
 					}
 				}
+				//循环动作
+				var Step2_RangeChrome = {{.T.lua.param_list_rangechrome}};
+				//先铺html
+				var Step2RangeChromeHtml = "";
+				for (var i in Step2_RangeChrome){
+					var action = Step2_RangeChrome[i].action
+					Step2RangeChromeHtml+=rangeActionHtmlMap[action];
+				}
+				$("#Step2 .chrome-range-actions").after(Step2RangeChromeHtml);
+				//再赋值
+				for (var i in Step2_RangeChrome){
+					var action = Step2_RangeChrome[i].action
+					var param = Step2_RangeChrome[i].param
+					var selector = Step2_RangeChrome[i].selector
+					$("#Step2 [name='Range.Chrome."+action+"']").val(param);
+					if (selector != ""){
+						$("#Step2 [sel='Range.Chrome."+action+".selector']").val(selector);
+					}
+				}
 			</script>
 		</fieldset>
 		<fieldset data-mode="guide">

src/web/templates/step3.html

@@ -14,26 +14,29 @@
 					</div>
 				</div>
 			</legend>
-			<p1>编辑动作集</p1>
-			<div class="form-group chrome-actions"  style="display: inline-block">
-				<i class="fa fa-plus-square" onclick="addChromeAction(this)"></i>
-<!--				<label class="control-label col-md-2 ">详情页地址：</label>-->
-<!--				<i class="fa fa-plus-square" onclick="addChromeAction(this)"></i>&nbsp;-->
-<!--				<i class="fa fa-times-circle" style="color: red;margin-top: 7px;"></i>-->
-<!--				<div class="col-md-5">-->
-<!--					<input type="text" class="form-control" placeholder="请输入地址" name="Chrome.navigate" />-->
-<!--				</div>-->
-<!--				<h6 style="color: red">（不可删除）</h6>-->
+			<div class="chrome-task-block">
+				<p1>基础动作</p1>
+				<div class="form-group chrome-base-actions chrome-actions"  style="display: inline-block">
+					<i class="fa fa-plus-square" onclick="addChromeAction(this)"></i>
+					<!--				<label class="control-label col-md-2 ">详情页地址：</label>-->
+					<!--				<i class="fa fa-plus-square" onclick="addChromeAction(this)"></i>&nbsp;-->
+					<!--				<i class="fa fa-times-circle" style="color: red;margin-top: 7px;"></i>-->
+					<!--				<div class="col-md-5">-->
+					<!--					<input type="text" class="form-control" placeholder="请输入地址" name="Chrome.navigate" />-->
+					<!--				</div>-->
+					<!--				<h6 style="color: red">（不可删除）</h6>-->
+				</div>
 			</div>
+
 			<script>
 				var Step3_Chrome = {{.T.lua.param_content_chrome}};
 				//先铺html
 				var Step3ChromeHtml = "";
 				for (var i in Step3_Chrome){
 					var action = Step3_Chrome[i].action
-					Step3ChromeHtml+=actionHtmlMap[action];
+					Step3ChromeHtml+=baseActionHtmlMap[action];
 				}
-				$("#Step3 .chrome-actions").after(Step3ChromeHtml);
+				$("#Step3 .chrome-base-actions").after(Step3ChromeHtml);
 				//再赋值
 				for (var i in Step3_Chrome){
 					var action = Step3_Chrome[i].action