修改

src/front/front.go

@@ -51,7 +51,7 @@ type Front struct {
 	runStep     xweb.Mapper `xweb:"/center/run"`             //方法测试
 	spiderPass  xweb.Mapper `xweb:"/center/spider/pass"`     //整体测试
 	runPinYin   xweb.Mapper `xweb:"/center/runpy"`           //获取拼音
-	saveStep    xweb.Mapper `xweb:"/center/save"`            //脚本保存
+	saveStep    xweb.Mapper `xweb:"/center/save"`            //爬虫保存
 	loadModel   xweb.Mapper `xweb:"/center/gmodel/(.*)"`     //加载模型
 	importdata  xweb.Mapper `xweb:"/center/importdata"`      //导入脚本
 	importfile  xweb.Mapper `xweb:"/center/importfile"`      //批量导入爬虫
@@ -541,7 +541,13 @@ func (f *Front) Importfile() {
 						o["name"] = cells[0].Value
 						o["code"] = cells[1].Value
 						o["channel"] = cells[2].Value
-						o["channeladdr"] = cells[3].Value
+						//重复域名的网站不再新增爬虫
+						href := cells[3].Value
+						one, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"href": href})
+						if len(*one) > 0 {
+							errorinfo[cells[1].Value] = "第" + strconv.Itoa(k) + "行爬虫" + cells[1].Value + ",该栏目已存在,请确认"
+						}
+						o["channeladdr"] = href
 						o["author"] = cells[4].Value
 						o["timestamp"] = time.Now().Unix()
 						o["status"] = 1
@@ -569,7 +575,6 @@ func (f *Front) Importfile() {
 						//table := cells[6].Value
 						//o["table"] = table
 						//o["transfercode"] = qu.IntAll(Transfercode[table])
-
 						query := map[string]interface{}{"code": cells[1].Value}
 						rs, _ := u.MgoEB.FindOne("import", query)
 						if len(*rs) > 0 {
@@ -657,6 +662,7 @@ func savelua(o map[string]interface{}) (bool, string) {
 	param["comeintime"] = time.Now().Unix()
 	param["code"] = o["code"]
 	param["site"] = o["name"]
+	param["href"] = o["channeladdr"]
 	param["channel"] = o["channel"]
 	param["createuser"] = (*one)["s_name"]
 	param["createuserid"] = id

src/front/spider.go

@@ -399,7 +399,10 @@ func (f *Front) SaveStep() {
 			} else if movevent, ok := util.Config.Uploadevents[f.OtherBase.SpiderMoveEvent].(string); ok && movevent != "" {
 				param["spidermovevent"] = movevent
 			}
-
+			//开发人员修改爬虫节点后，在审核人员上架时，要在原来的节点下架，临时记录要下架的节点downevent
+			if event := qu.IntAll((*one)["event"]); event != tmpEvent && event != 7000 {
+				param["downevent"] = event
+			}
 			//三级页复制
 			param["str_recontent"] = f.StepRe3.Expert
 			param["iscopycontent"] = f.StepRe3.Checked
@@ -522,7 +525,7 @@ func (f *Front) RunStep() {
 	}
 	if f.Method() == "POST" {
 		switch f.GetString("step") {
-		case "step1":
+		case "step1": //publishtime
 			ptime := []interface{}{
 				f.Step1.DateFormat,
 				f.Step1.Address,
@@ -537,7 +540,7 @@ func (f *Front) RunStep() {
 			if err == nil {
 				f.ServeJson(rs)
 			}
-		case "step2":
+		case "step2": //list
 			addrs := strings.Split(f.Step2.Listadds, "\n")
 			if len(addrs) > 0 {
 				for k, v := range addrs {
@@ -577,7 +580,7 @@ func (f *Front) RunStep() {
 			} else if err.(error).Error() == "no" {
 				f.ServeJson(rs[0])
 			}
-		case "step3":
+		case "step3": //detail
 			content := []interface{}{
 				f.Step3.ContentChooser,
 				f.Step3.ElementChooser,
@@ -616,45 +619,58 @@ func (f *Front) RunStep() {
 
 //爬虫测试数据json
 func (f *Front) GetJson() {
-	list, _ := f.GetSession("listInfo").([]map[string]interface{})
-	data, _ := f.GetSession("dataInfo").(map[string]interface{})
-	descript := f.GetSession("task_descript")
-	remark := f.GetSession("task_remark")
-	rateremark := f.GetSession("task_rateremark")
-	reason := f.GetSession("reason")
+	code := f.GetString("code")
 	username := f.GetSession("username").(string)
-	msg := f.GetSession(username + "_msg")
-	if len(data) > 0 {
-		data["contenthtml"] = ""
-	}
-	for k, v := range list {
-		v["a_index"] = k + 1
+	if tr := TestResultMap[username+code]; tr != nil {
+		task, _ := u.MgoEB.FindOne("task", map[string]interface{}{"code": code, "i_state": 3})
+		comeintime := int64(0)
+		if len(*task) > 0 {
+			comeintime = qu.Int64All((*task)["l_comeintime"])
+			tr.task_remark = "审核任务创建时间：" + qu.FormatDateByInt64(&comeintime, qu.Date_Short_Layout) + ";" + tr.task_remark
+		}
+		data := tr.dataInfo
+		list := tr.listInfo
+		if len(data) > 0 {
+			data["contenthtml"] = ""
+		}
+		for k, v := range list {
+			v["a_index"] = k + 1
+		}
+
+		f.T["list"] = list
+		f.T["data"] = data
+		f.T["descript"] = tr.task_descript
+		f.T["remark"] = tr.task_remark
+		f.T["rateremark"] = tr.task_rateremark
+		f.T["reason"] = tr.reason
+		f.T["msg"] = tr.msg
+		f.T["comeintime"] = comeintime
+		delete(TestResultMap, username+code)
 	}
 
-	f.T["list"] = list
-	f.T["data"] = data
-	f.T["descript"] = descript
-	f.T["remark"] = remark
-	f.T["rateremark"] = rateremark
-	f.T["reason"] = reason
-	f.T["msg"] = msg
+	f.Render("jsonInfo.html", &f.T)
+}
 
-	f.DelSession("listInfo")
-	f.DelSession("dataInfo")
-	f.DelSession("task_descript")
-	f.DelSession("task_remark")
-	f.DelSession("task_rateremark")
-	f.DelSession("reason")
+var TestResultMap = map[string]*TestResult{} //username+code
 
-	f.Render("jsonInfo.html", &f.T)
+//某个爬虫整体测试结果
+type TestResult struct {
+	task_remark     string
+	task_rateremark []string
+	task_descript   string
+	reason          string
+	msg             string
+	listInfo        []map[string]interface{}
+	dataInfo        map[string]interface{}
 }
 
 //整体测试
 func (f *Front) SpiderPass() {
 	defer mu.Catch()
+	tr := &TestResult{}
 	list := []map[string]interface{}{}
 	data := map[string]interface{}{}
-	msg1 := ""
+	msgArr := []string{}
 	code := f.GetString("code")
 	downloadnode := f.GetString("node")
 	//根据code查询待确认任务
@@ -685,16 +701,15 @@ func (f *Front) SpiderPass() {
 		remark = ""
 		remark = strings.Join(remarktmp, "")
 	}
+	tr.task_remark = remark
+	tr.task_rateremark = rateremarktmp
+	tr.task_descript = descript
 
-	f.SetSession("task_remark", remark)
-	f.SetSession("task_rateremark", rateremarktmp)
-	f.SetSession("task_descript", descript)
 	//基本信息、方法一、方法二、方法三、总请求次数、go方法一、go方法二、go方法三、列表页条数
 	steps := []interface{}{false, false, false, false, 0, 0, 0, 0, 0}
-	//one, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": code})
 	one, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
 	reason, _ := (*one)["reason"].(string)
-	f.SetSession("reason", reason)
+	tr.reason = reason
 	if len(*one) > 0 && (*one)["oldlua"] == nil {
 		common := (*one)["param_common"].([]interface{})
 		if len(common) < 13 {
@@ -713,17 +728,29 @@ func (f *Front) SpiderPass() {
 		script = (*one)["luacontent"].(string)
 	}
 	if liststr != "" && contentstr != "" {
-		msg1 = u.SpiderPassCheckLua(liststr, contentstr, (*one)) //校验
+		msgArr = u.SpiderPassCheckLua(liststr, contentstr, (*one)) //校验
 	}
 	s := spider.CreateSpider(downloadnode, script)
 	s.SpiderMaxPage = 1
 	s.Timeout = 60
-	time, timeerr := s.GetLastPublishTime()
-	if timeerr == nil && len(time) > 4 {
+	timestr, timeerr := s.GetLastPublishTime()
+	if timeerr == nil && len(timestr) > 4 {
 		steps[1] = true
 		list, _ = s.DownListPageItem()
+		for _, l := range list {
+			if publishtime := qu.ObjToString(l["publishtime"]); publishtime == "0" || publishtime == "" {
+				msgArr = append(msgArr, "列表页publishtime取值异常")
+				break
+			} else {
+				t, err := time.ParseInLocation(qu.Date_Full_Layout, publishtime, time.Local)
+				if err != nil || t.Unix() <= 0 {
+					msgArr = append(msgArr, "列表页publishtime取值异常")
+					break
+				}
+			}
+		}
 		if len(list) > 0 {
-			f.SetSession("listInfo", list)
+			tr.listInfo = list
 			listone := list[0]
 			if len(qu.ObjToString(listone["href"])) < 7 ||
 				(qu.ObjToString(listone["publishtime"]) != "0" && len(qu.ObjToString(listone["publishtime"])) < 5) ||
@@ -744,9 +771,7 @@ func (f *Front) SpiderPass() {
 						data = map[string]interface{}{}
 						s.DownloadDetailPage(param, data)
 						if len(data) > 0 {
-							f.SetSession("dataInfo", data)
-						} else {
-							f.SetSession("dataInfo", "")
+							tr.dataInfo = data
 						}
 						if len(data) == 0 || data["detail"].(string) == "" {
 							steps[3] = false
@@ -758,9 +783,6 @@ func (f *Front) SpiderPass() {
 					steps[3] = true
 				}
 			}
-		} else {
-			f.SetSession("listInfo", "")
-			f.SetSession("dataInfo", "")
 		}
 	}
 	//关闭laustate
@@ -771,11 +793,10 @@ func (f *Front) SpiderPass() {
 	steps[7] = s.Test_goreqcon
 	//校验
 	msg := u.SpiderPassCheckListAndDetail(list, data)
-	if msg1 != "" {
-		msg = msg1 + "；" + msg
-	}
+	msgArr = append(msgArr, msg...)
 	username := f.GetSession("username").(string)
-	f.SetSession(username+"_msg", msg)
+	tr.msg = strings.Join(msgArr, "；")
+	TestResultMap[username+code] = tr
 	f.ServeJson(steps)
 }
 
@@ -1060,6 +1081,7 @@ func UpStateAndUpSpider(code, id, reason, username string, state int) (bool, err
 	}
 	if len(*one) > 0 {
 		var event int
+		unset := map[string]interface{}{}
 		if (*one)["event"] != nil {
 			event = qu.IntAll((*one)["event"])
 		} else {
@@ -1081,7 +1103,13 @@ func UpStateAndUpSpider(code, id, reason, username string, state int) (bool, err
 			// }
 			upresult, err = spider.UpdateSpiderByCodeState(code, fmt.Sprint(state), event) //下架
 			qu.Debug("下架:", upresult, code)
-		case Sp_state_5: //上架(爬虫端在更新上架的时候为了更新内存中字段，采用先下架在上架)
+		case Sp_state_5: //上架(爬虫端在更新上架的时候为了更新内存中字段，采用先下架上架)
+			if downevent := qu.IntAll((*one)["downevent"]); downevent != 0 { //爬虫开发修改爬虫节点，审核人员上架爬虫时，原来爬虫所在节点下架
+				upresult, err = spider.UpdateSpiderByCodeState(code, "6", downevent)
+				if upresult && err == nil {
+					unset = map[string]interface{}{"downevent": ""}
+				}
+			}
 			upresult, err = spider.UpdateSpiderByCodeState(code, "6", event)
 			qu.Debug("下架:", upresult, code)
 			if upresult && err == nil {
@@ -1101,7 +1129,13 @@ func UpStateAndUpSpider(code, id, reason, username string, state int) (bool, err
 			upset := map[string]interface{}{"state": state} //修改状态
 			if (*one)["oldlua"] != nil {                    //老脚本上传
 				//upresult = u.MgoE.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": upset}, true, false)
-				upresult = u.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": upset}, true, false)
+				up := map[string]interface{}{
+					"$set": upset,
+				}
+				if len(unset) > 0 {
+					up["$unset"] = unset
+				}
+				upresult = u.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, up, true, false)
 			} else {
 				if state == Sp_state_1 { //提交审核
 					upset["l_complete"] = time.Now().Unix()
@@ -1127,8 +1161,14 @@ func UpStateAndUpSpider(code, id, reason, username string, state int) (bool, err
 					//upset["modifytime"] = time.Now().Unix()
 					upset["l_uploadtime"] = time.Now().Unix() //l_complete爬虫完成时间
 				}
+				up := map[string]interface{}{
+					"$set": upset,
+				}
+				if len(unset) > 0 {
+					up["$unset"] = unset
+				}
 				//upresult = u.MgoE.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": upset}, false, false)
-				upresult = u.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": upset}, false, false)
+				upresult = u.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, up, false, false)
 				qu.Debug("提交日志：", code, upset, upresult)
 				if upresult && (state == Sp_state_2 || state == Sp_state_3) { //打回、审核记录日志
 					types := "打回"

src/main.go

@@ -9,13 +9,11 @@ import (
 
 	//. "luaweb/task"
 	"luaerrdata"
-	"taskManager"
-	"timetask"
-	"udp"
-
 	"net/http"
 	qu "qfw/util"
 	"qfw/util/redis"
+	"taskManager"
+	"timetask"
 	u "util"
 
 	codegrpc "analysiscode"
@@ -88,7 +86,7 @@ func init() {
 		qu.ObjToString(util.Config.OssInfo["ossBucketName"]),
 	)
 	//udp
-	udp.InitUdp()
+	//udp.InitUdp()//异常数据重采，暂取消该功能
 }
 
 //

src/spider/script.go

@@ -13,9 +13,11 @@ import (
 	"crypto/aes"
 	"encoding/base64"
 	"encoding/json"
+	"fmt"
 	"github.com/shopspring/decimal"
 	"io/ioutil"
 	mu "mfw/util"
+	"mime/multipart"
 	"net/http"
 	"net/url"
 	"path"
@@ -620,6 +622,78 @@ func (s *Script) LoadScript(downloadnode, script string, isfile ...string) {
 		S.Push(lua.LString(contentHtml))
 		return 1
 	}))
+	//辽宁省招标投标监管网
+	s.L.SetGlobal("multipartRequest", s.L.NewFunction(func(S *lua.LState) int {
+		yzm := S.ToString(-5)
+		verifyId := S.ToString(-4)
+		id := S.ToString(-3)
+		param := S.ToTable(-2)
+		head := S.ToTable(-1)
+		paramMap := util.GetTable(param)
+		headMap := util.GetTable(head)
+		qu.Debug(paramMap)
+		qu.Debug(headMap)
+		paramMap = map[string]interface{}{
+			"baseUrl":        "https://www.lntb.gov.cn/mhback",
+			"realRequestUri": "/api/cTenderNoticeController/getDetail/" + id,
+			"verifyId":       verifyId,
+			"result":         yzm,
+		}
+		headMap = map[string]interface{}{
+			"Referer":          "",
+			"sec-ch-ua-mobile": "?0",
+			"User-Agent":       "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36",
+		}
+		//由于验证码识别不准确，加减法呢特殊处理
+		for _, w := range []string{"+", "-"} {
+			if arr := strings.Split(yzm, w); len(arr) == 2 {
+				if w == "+" {
+					yzm = fmt.Sprint(qu.IntAll(arr[0]) + qu.IntAll(arr[1]))
+				} else {
+					yzm = fmt.Sprint(qu.IntAll(arr[0]) - qu.IntAll(arr[1]))
+				}
+			}
+		}
+		qu.Debug(yzm, verifyId, id)
+		payload := &bytes.Buffer{}
+		writer := multipart.NewWriter(payload)
+		//param
+		for k, v := range paramMap {
+			_ = writer.WriteField(k, v.(string))
+		}
+		err := writer.Close()
+		if err != nil {
+			qu.Debug(err)
+			return 1
+		}
+		//request
+		client := &http.Client{}
+		req, err := http.NewRequest("POST", "https://www.lntb.gov.cn/mhback/kk-anti-reptile/validate", payload)
+		//header
+		for k, v := range headMap {
+			req.Header.Add(k, v.(string))
+		}
+		req.Header.Set("Content-Type", writer.FormDataContentType())
+		res, err := client.Do(req)
+		if err != nil {
+			qu.Debug(err)
+			return 1
+		}
+		defer res.Body.Close()
+		body, err := ioutil.ReadAll(res.Body)
+		if err != nil {
+			fmt.Println(err)
+			return 1
+		}
+		result := false
+		resultMap := map[string]bool{}
+		if json.Unmarshal(body, &resultMap) == nil {
+			result = resultMap["result"]
+		}
+		fmt.Println(string(body), resultMap)
+		S.Push(lua.LBool(result))
+		return 1
+	}))
 }
 
 //

src/spider/service.go

@@ -166,7 +166,8 @@ func GetPageList(comm, param []interface{}, model map[string]interface{}, listch
 		return nil, err
 	}
 	sp := CreateSpider(downloadnode, script)
-	sp.SpiderMaxPage = 1
+	sp.SpiderMaxPage = sp.SpiderStartPage
+	//sp.SpiderMaxPage = 1
 	defer sp.L.Close()
 	list, err = sp.DownListPageItemTest()
 	return

src/timetask/timetask.go

@@ -104,17 +104,17 @@ func UpdateSiteInfo() {
 	sites, _ := util.MgoEB.Find(sp.Config.SiteColl, ``, ``, `{"site":1}`, false, -1, -1)
 	for _, s := range *sites {
 		site := qu.ObjToString(s["site"])
-		domain, status, event, platform, area, city, district, _ := util.GetLuasInfoBySite(site)
+		domain, status, event, platform, _, _, _, _ := util.GetLuasInfoBySite(site)
 		set := map[string]interface{}{
 			"$set": map[string]interface{}{
 				"platform":      platform,
 				"event":         event,
 				"spider_status": status,
 				"domain":        domain,
-				"area":          area,
-				"city":          city,
-				"district":      district,
-				"updatetime":    time.Now().Unix(),
+				//"area":          area,
+				//"city":          city,
+				//"district":      district,
+				"updatetime": time.Now().Unix(),
 			},
 		}
 		util.MgoEB.UpdateById(sp.Config.SiteColl, s["_id"], set)

src/util/util.go

@@ -119,8 +119,7 @@ func InitAreaCity() {
 }
 
 //爬虫整体测试时校验爬虫代码
-func SpiderPassCheckLua(liststr, contentstr string, lua map[string]interface{}) string {
-	msg := []string{}
+func SpiderPassCheckLua(liststr, contentstr string, lua map[string]interface{}) (msg []string) {
 	//校验含过滤方法stringFind但没有过滤注释“--关键词过滤”
 	if strings.Contains(liststr, "stringFind") && !strings.Contains(liststr, "--关键词过滤") {
 		msg = append(msg, "列表页代码有过滤方法stringFind但缺少注释：--关键词过滤")
@@ -128,6 +127,12 @@ func SpiderPassCheckLua(liststr, contentstr string, lua map[string]interface{})
 	if strings.Contains(contentstr, "--关键词过滤") && !strings.Contains(contentstr, "delete") {
 		msg = append(msg, `三级页代码有过滤方法但缺少data["delete"]="true"`)
 	}
+	if !strings.Contains(contentstr, "s_title") {
+		msg = append(msg, "三级页缺少s_title")
+	}
+	if !strings.Contains(contentstr, "getFileAttachmentsArrayWithTag") && !strings.Contains(contentstr, "downloadFile") {
+		msg = append(msg, "三级页缺少下载附件方法")
+	}
 	//1.检测spidercode、site、channel
 	if param, ok := lua["param_common"].([]interface{}); ok && len(param) >= 3 {
 		spidercode := qu.ObjToString(param[0])
@@ -171,12 +176,11 @@ func SpiderPassCheckLua(liststr, contentstr string, lua map[string]interface{})
 	if !strings.Contains(liststr, "sendListNum") {
 		msg = append(msg, "sendListNum方法缺失")
 	}
-	return strings.Join(msg, ",")
+	return
 }
 
 //爬虫整体测试时校验列表页和详情页内容
-func SpiderPassCheckListAndDetail(list []map[string]interface{}, data map[string]interface{}) string {
-	msg := []string{}
+func SpiderPassCheckListAndDetail(list []map[string]interface{}, data map[string]interface{}) (msg []string) {
 	if len(list) > 0 {
 		p_zero := 0
 		h_flag := true
@@ -228,7 +232,7 @@ func SpiderPassCheckListAndDetail(list []map[string]interface{}, data map[string
 			}
 		}
 	}
-	return strings.Join(msg, ",")
+	return
 }
 
 func GetLuasInfoBySite(site string) (domain, status, event, platform, area, city, district string, remarktime int64) {

src/web/staticres/js/common.js

@@ -272,7 +272,7 @@ common.spider.setModel=function(name,p){
 			var label=$("<label class='col-sm-4 control-label'>"+model[k]+"</label>");
 			var c_div=$("<div class='col-md-7'></div>");
 			if(model[k]=="省份"){
-				var select=$("<select class='form-control' onchange='getcity($(this).val())'></select>")
+				var select=$("<select class='form-control' disabled onchange='getcity($(this).val())'></select>")
 				select.attr("name",k);
 				select.attr("id",k);
 				for(var i in common.spider.json){
@@ -284,13 +284,16 @@ common.spider.setModel=function(name,p){
 				}
 				modelbody.append(div.append(label).append(c_div.append(select)));
 			}else if(model[k]=="城市"){
-				var select=$("<select class='form-control' onchange='cleardistrict()'></select>")
+				var select=$("<select class='form-control' disabled onchange='cleardistrict()'></select>")
 				select.attr("name",k);
 				select.attr("id",k);
 				modelbody.append(div.append(label).append(c_div.append(select)));
 			}else{
-				var input=$("<input type='text' 	class='form-control'/>");
+				var input=$("<input type='text'  class='form-control'/>");
 				input.attr("name",k);
+				if(k == "district"){
+					input.attr("disabled",true);
+				}
 				input.attr("id",k);
 				input.attr("placeholder",model[k]);
 				modelbody.append(div.append(label).append(c_div.append(input)));

src/web/templates/index.html

@@ -417,7 +417,6 @@ $(function(){
 			}},
 			{{if eq (session "platform") "python"}}
 			{ "data": "grade",render:function(val){
-				console.log(val)
 				if(val == undefined||val == 0){
 					return "普通"
 				}else if (val == 1){
@@ -818,7 +817,6 @@ $(function(){
 				if($(this).prop("checked")){
 					var codePlatform = $(this).attr("platform");//被分配爬虫所属平台
 					var userScope = $(this).attr("scope");//被分配爬虫所属平台
-					console.log(userPlatform,codePlatform,userScope)
 					//lua平台除外包人员外，内部不可分配爬虫
 					if (codePlatform == "golua平台" && userPlatform == "golua平台" &&  userScope == '1'){
 						nocodes.push($(this).attr("code"));

src/web/templates/lualist.html

@@ -107,24 +107,25 @@
                             return  vals
                         }},
                     { "data": "param_common",render:function(val,a,row){
-                            var site="",chanel="",code="",href="";
-                            if(val.length>2){
-                                code=val[0]
-                                site=val[1]
-                                chanel=val[2]
-                                href="http://test.qmx.top:3000/dashboard/db/pa-chong-cai-ji-fen-jiao-ben-30ri-zou-shi-fen-xi?orgId=5&var-spidercode="+code+"&var-site="+site+"&=var-channel="+chanel;
-                            }else{
-                                code=row.code
-                                site=val[1]
-                                href="/center/spider/edit/"+code+"__";//老爬虫
-                            }
-                            var codeshort=""
-                            if(code.length>33){
-                                codeshort=code.substr(0,33)+"...";
-                            }else{
-                                codeshort=code
-                            }
-                            return "<a href='"+href+"' target='_blank' title='"+code+"'>"+codeshort+"</a>"
+                        return row.code;
+                            // var site="",chanel="",code="",href="";
+                            // if(val.length>2){
+                            //     code=val[0]
+                            //     site=val[1]
+                            //     chanel=val[2]
+                            //     href="http://test.qmx.top:3000/dashboard/db/pa-chong-cai-ji-fen-jiao-ben-30ri-zou-shi-fen-xi?orgId=5&amp;var-spidercode="+code+"&amp;var-site="+site+"&amp;=var-channel="+chanel;
+                            // }else{
+                            //     code=row.code
+                            //     site=val[1]
+                            //     href="/center/spider/edit/"+code+"__";//老爬虫
+                            // }
+                            // var codeshort=""
+                            // if(code.length>33){
+                            //     codeshort=code.substr(0,33)+"...";
+                            // }else{
+                            //     codeshort=code
+                            // }
+                            // return "<a href='"+href+"' target='_blank' title='"+code+"'>"+codeshort+"</a>"
                         }},
                     { "data": "createuser"},
                     { "data": "l_uploadtime",render:function(val,a,row){
@@ -224,7 +225,6 @@
                     }
                     //平台
                     var platform = $("#platform").val();
-                    console.log(platform)
                     if(platform){
                         e.platform=platform
                     }else{

src/web/templates/luamovelist.html

@@ -303,7 +303,6 @@
 		return "<select onchange='updateesp(this.value,\""+id+"\")' class='form-control input-sm'>"+spiderEvent+"</select>"
 	};
 	function updateesp(event,id){
-		console.log(event,id);
 		$.ajax({
 			url:"/center/luamove/updatetoevent",
 			type:"post",

src/web/templates/sitelist.html

@@ -483,7 +483,6 @@ function del(){
 			ids.push($(this).val());
 		}
 	});
-  console.log(ids)
   if(ids.length>0){
     siteDelete(ids.join(","));
     $("#selrow").prop('checked',false);

src/web/templates/sitelualist.html

@@ -33,7 +33,6 @@
   </section>
 
   <script>
-    console.log()
     $(function(){
       lualisttable=$('#spider').DataTable({
         "language": {

src/web/templates/spiderbase.html

@@ -184,7 +184,7 @@
 							增量节点
 						</label>
 						<div class="col-sm-7">
-							<select name="OtherBase.SpiderMoveEvent" disabled id="spidermovevent" class="form-control">
+							<select name="OtherBase.SpiderMoveEvent" id="spidermovevent" class="form-control">
 <!--								<option value="bid" disabled style="color:#DADADA">-->
 <!--									高性能模式-->
 <!--								</option>-->
@@ -205,7 +205,6 @@
 								// }
 								var startext = "<option";
 								var events = {{.T.events}};
-								console.log({{.T.spidertype}})
 								for(k in events){
 									var event = events[k];
 									var endtext = " value="+event+">"+event+"</option>";

src/web/templates/spideredit.html

@@ -126,7 +126,7 @@
 						<input type="radio" name="jd-0"  value="bid">线路2</input>
 					</div>
 		  </div>
-      <a type="button" class="hidden" target="_blank" href="/center/spider/json" id="jsonClick"></a>
+      <a type="button" class="hidden" target="_blank" href="/center/spider/json?code={{.T.lua.code}}" id="jsonClick"></a>
 				<div class="box box-solid spidernamelist hide">
 				<div class="box-header with-border">
 					<b>参考爬虫(点击选取)</b>
@@ -321,7 +321,7 @@
 				editor_2.refresh();
 				editor_3.refresh();
 				editor_4.refresh();
-        editor_re3.refresh();
+        		editor_re3.refresh();
 			}
 			
 		})
@@ -332,7 +332,7 @@
 			
 		})
 		$("legend a").click(function(event,editor){
-      var node = $(this).parent().find("input[type='radio']:checked").val()
+      		var node = $(this).parent().find("input[type='radio']:checked").val()
 			var target=$(this).attr("for");
 			var check=function(me){
 				var name=$(me).attr("name")
@@ -362,8 +362,14 @@
 					data["imodal"]=0;
 					data["script"]=true;
 				}
-        //data中加入选择的节点
-        data["downloadnode"]= node
+				//data中加入选择的节点
+				data["downloadnode"]= node
+				//测试页码
+				if (data["step"] == "step2"){
+					data["Base.SpiderStartPage"] = data["pageno"]
+				}
+				// console.log(data)
+				// return;
 				common.spider.test(data,function(me){
 					common.maskHide();
 					if(editor){

src/web/templates/spiderview.html

@@ -159,8 +159,12 @@
 							data["imodal"]=0;
 							data["script"]=true;
 						}
-            //data中加入选择的节点
-            data["downloadnode"]= node
+						//data中加入选择的节点
+						data["downloadnode"]= node
+						//测试页码
+						if (data["step"] == "step2"){
+							data["Base.SpiderStartPage"] = data["pageno"]
+						}
 						common.spider.test(data,function(me){
 							common.maskHide();
 							if(editor){

src/web/templates/step2.html

@@ -1,90 +1,105 @@
 <div class="tab-pane" id="tab_2">
 	<form class="form-horizontal" id="step2">
 		<fieldset data-mode="guide">
-				<legend>
-         <div style="display: flex;flex-direction: row;align-items: center;">
-            第二步&nbsp;&nbsp;<a class="btn btn-default btn-sm" for="step2">测&nbsp;&nbsp;试</a>
-            <div style="display: flex;flex-direction: row;align-items: center;">
-                <span style="margin-left: 10px;margin-right: 10px;">线路:</span>
-								<div style="font-size: 16px;">
-									<input type="radio" checked name="jd-2-1" value="test">默认</input>
-									<input type="radio" name="jd-2-1" value="comm">线路1</input>
-									<input type="radio" name="jd-2-1" value="bid">线路2</input>
-								</div>
-					  </div>
-          </div>
-        </legend>
-					<div class="form-group">
-						<label class="control-label col-md-3 ">列表页地址：</label>
-						<div class="col-md-6">
-							<input type="text" class="form-control" placeholder="http://www.ccgp.gov.cn/zycg/zycgdt/index#pageno#.htm" name="Step2.Listadd" value="{{if .T.lua.param_list}}{{index .T.lua.param_list 0}}{{end}}" />
-						</div>
-					</div>
-					
-					<div class="form-group fixurl ">
-						<label class="control-label col-md-3 ">固定地址：</label>
-						<div class="col-md-6">
-							<textarea class="form-control" rows="3" placeholder="每行输入一条" name="Step2.Listadds">{{if .T.lua.param_list}}{{index .T.lua.param_list 1}}{{end}}</textarea>
-						</div>
-					</div>
-					
-					<div class="form-group">
-						<label class="control-label col-md-3 ">信息块选择器：</label>
-						<div class="col-md-6">
-							<input type="text" class="form-control" placeholder="ul.list li" name="Step2.BlockChooser" value="{{if .T.lua.param_list}}{{index .T.lua.param_list 2}}{{end}}" />
-						</div>
-					</div>
-					
-					<div class="form-group">
-						<label class="control-label col-md-3 ">信息地址选择器：</label>
-						<div class="col-md-6">
-							<input type="text" class="form-control" placeholder="a:attr(href)" name="Step2.AddressChooser" value="{{if .T.lua.param_list}}{{index .T.lua.param_list 3}}{{end}}" />
-						</div>
-					</div>
-					
-					<div class="form-group">
-						<label class="control-label col-md-3 ">信息标题选择器：</label>
-						<div class="col-md-6">
-							<input type="text" class="form-control" placeholder="a"  name="Step2.TitleChooser" value="{{if .T.lua.param_list}}{{index .T.lua.param_list 4}}{{end}}" />
-						</div>
-					</div>
-					
-					<div class="form-group">
-						<label class="control-label col-md-3 ">信息时间选择器：</label>
-						<div class="col-md-6">
-							<input type="text" class="form-control" placeholder="span" name="Step2.DateChooser" value="{{if .T.lua.param_list}}{{index .T.lua.param_list 5}}{{end}}" />
-						</div>
-					</div>
-					
-					<div class="form-group">
-						<label class="control-label col-md-3 ">时间格式：</label>
-						<div class="col-md-6">
-							<select class="form-control" name="Step2.DateFormat" id="Step2DateFormat">
-								<option value="yyyyMMddHHmmss" >年月日时分秒</option>
-								<option value="yyyyMMddHHmm" >年月日时分</option>
-								<option value="yyyyMMdd" >年月日</option>
-								<option value="MMdd" >月日</option>
-							</select>
-						</div>
-						<script>
-							var v="{{if .T.lua.param_list}}{{index .T.lua.param_list 6}}{{end}}"
-							if(v!=""){
-								$("#Step2DateFormat").val(v);
-							}
-						</script>
-					</div>
-					<div class="form-group">
-						<label class="control-label col-md-3">改版检测：</label>
-						<div class="col-md-6">
-							<textarea class="form-control" rows="3" placeholder="格式：选择器==结果,每行一条。 如:list:eq(0).attr('id')==listid" name="listcheck">{{if .T.lua.listcheck}}{{index .T.lua.listcheck}}{{end}}</textarea>
+			<legend>
+				<div style="display: flex;flex-direction: row;align-items: center;">
+					第二步&nbsp;&nbsp;<a class="btn btn-default btn-sm" for="step2">测&nbsp;&nbsp;试</a>
+					<div style="display: flex;flex-direction: row;align-items: center;">
+						<span style="margin-left: 10px;margin-right: 10px;">线路:</span>
+						<div style="font-size: 16px;">
+							<input type="radio" checked name="jd-2-1" value="test">默认</input>
+							<input type="radio" name="jd-2-1" value="comm">线路1</input>
+							<input type="radio" name="jd-2-1" value="bid">线路2</input>
 						</div>
 					</div>
+				</div>
+        	</legend>
+			<div class="form-group">
+				<label class="control-label col-md-3 ">列表页地址：</label>
+				<div class="col-md-6">
+					<input type="text" class="form-control" placeholder="http://www.ccgp.gov.cn/zycg/zycgdt/index#pageno#.htm" name="Step2.Listadd" value="{{if .T.lua.param_list}}{{index .T.lua.param_list 0}}{{end}}" />
+				</div>
+			</div>
+
+			<div class="form-group fixurl ">
+				<label class="control-label col-md-3 ">固定地址：</label>
+				<div class="col-md-6">
+					<textarea class="form-control" rows="3" placeholder="每行输入一条" name="Step2.Listadds">{{if .T.lua.param_list}}{{index .T.lua.param_list 1}}{{end}}</textarea>
+				</div>
+			</div>
+
+			<div class="form-group">
+				<label class="control-label col-md-3 ">信息块选择器：</label>
+				<div class="col-md-6">
+					<input type="text" class="form-control" placeholder="ul.list li" name="Step2.BlockChooser" value="{{if .T.lua.param_list}}{{index .T.lua.param_list 2}}{{end}}" />
+				</div>
+			</div>
+
+			<div class="form-group">
+				<label class="control-label col-md-3 ">信息地址选择器：</label>
+				<div class="col-md-6">
+					<input type="text" class="form-control" placeholder="a:attr(href)" name="Step2.AddressChooser" value="{{if .T.lua.param_list}}{{index .T.lua.param_list 3}}{{end}}" />
+				</div>
+			</div>
+
+			<div class="form-group">
+				<label class="control-label col-md-3 ">信息标题选择器：</label>
+				<div class="col-md-6">
+					<input type="text" class="form-control" placeholder="a"  name="Step2.TitleChooser" value="{{if .T.lua.param_list}}{{index .T.lua.param_list 4}}{{end}}" />
+				</div>
+			</div>
+
+			<div class="form-group">
+				<label class="control-label col-md-3 ">信息时间选择器：</label>
+				<div class="col-md-6">
+					<input type="text" class="form-control" placeholder="span" name="Step2.DateChooser" value="{{if .T.lua.param_list}}{{index .T.lua.param_list 5}}{{end}}" />
+				</div>
+			</div>
+
+			<div class="form-group">
+				<label class="control-label col-md-3 ">时间格式：</label>
+				<div class="col-md-6">
+					<select class="form-control" name="Step2.DateFormat" id="Step2DateFormat">
+						<option value="yyyyMMddHHmmss" >年月日时分秒</option>
+						<option value="yyyyMMddHHmm" >年月日时分</option>
+						<option value="yyyyMMdd" >年月日</option>
+						<option value="MMdd" >月日</option>
+					</select>
+				</div>
+				<script>
+					var v="{{if .T.lua.param_list}}{{index .T.lua.param_list 6}}{{end}}"
+					if(v!=""){
+						$("#Step2DateFormat").val(v);
+					}
+				</script>
+			</div>
+			<div class="form-group">
+				<label class="control-label col-md-3">改版检测：</label>
+				<div class="col-md-6">
+					<textarea class="form-control" rows="3" placeholder="格式：选择器==结果,每行一条。 如:list:eq(0).attr('id')==listid" name="listcheck">{{if .T.lua.listcheck}}{{index .T.lua.listcheck}}{{end}}</textarea>
+				</div>
+			</div>
 		</fieldset>
 		<fieldset data-mode="expert" class="hide">
 				<legend>
-         <div style="display: flex;flex-direction: row;align-items: center;flex-wrap: wrap;">
+         	<div style="display: flex;flex-direction: row;align-items: center;flex-wrap: wrap;">
            第二步（专家模式）&nbsp;&nbsp;<a class="btn btn-default btn-sm" for="step2">测&nbsp;&nbsp;试</a>
-            <div style="display: flex;flex-direction: row;align-items: center;">
+				<div style="font-size: 16px;">
+					<span style="margin-left: 10px">页码:</span>
+					<select name="pageno">
+						<option value="1">1</option>
+						<option value="2">2</option>
+						<option value="3">3</option>
+						<option value="4">4</option>
+						<option value="5">5</option>
+						<option value="6">6</option>
+						<option value="7">7</option>
+						<option value="8">8</option>
+						<option value="9">9</option>
+						<option value="10">10</option>
+					</select>
+				</div>
+				<div style="display: flex;flex-direction: row;align-items: center;">
                 <span style="margin-left: 10px;margin-right: 10px;">线路:</span>
 								<div style="font-size: 16px;">
 									<input type="radio" checked name="jd-2-2" value="test">默认</input>

src/web/templates/task.html

@@ -169,10 +169,11 @@
 					return vals
 				}},
 				{"data": "s_code",render:function(val,a,row){	
-					var	site = row["s_site"];
-					var	chanel = row["s_channel"];
-					var	href = "http://test.qmx.top:3000/dashboard/db/pa-chong-cai-ji-fen-jiao-ben-30ri-zou-shi-fen-xi?orgId=5&var-spidercode="+val+"&var-site="+site+"&=var-channel="+chanel;	
-					return "<a href='"+href+"' target='_blank' title='"+val+"'>"+val+"</a>"
+					// var	site = row["s_site"];
+					// var	chanel = row["s_channel"];
+					// var	href = "http://test.qmx.top:3000/dashboard/db/pa-chong-cai-ji-fen-jiao-ben-30ri-zou-shi-fen-xi?orgId=5&amp;var-spidercode="+val+"&amp;var-site="+site+"&amp;=var-channel="+chanel;
+					// return "<a href='"+href+"' target='_blank' title='"+val+"'>"+val+"</a>"
+						return val;
 				}},
 				{"data": "i_state","width":"5%"},
 				{"data": "s_type","width":"7%",render:function(val,a,row){

src/web/templates/vps/vpslist.html

@@ -348,7 +348,6 @@
 			let vpsimage = $("#vpsimage").val();
 			let vpsnum = $("#vpsnum").val();
 			let vpsduration = $("#vpsduration").val();
-			console.log(vpsname,vpsimage,vpsnum,vpsduration)
 			if (vpsname == ""||vpsnum == ""||vpsduration == ""){
 				alert("请填写完整的信息！")
 				return