mxs 1 рік тому
батько
коміт
a762ec47ea

+ 1 - 1
src/config.json

@@ -101,7 +101,7 @@
     },
     "fileServer": "http://123.56.236.148:9333",
     "jsvmurl": "http://127.0.0.1:8080/jsvm",
-    "renderaddr": "http://59.110.6.43:8998/render.json",
+    "renderaddr": "http://192.168.3.182:8050/render.json",
     "proxyaddr": "http://cc.spdata.jianyu360.com/crawl/proxy/socks5/fetch",
     "proxyauthor": "Basic amlhbnl1MDAxOjEyM3F3ZSFB",
     "pwaddr": "http://pw.spdata.jianyu360.com/web/intercept/request?url=%s",

+ 0 - 2
src/front/front.go

@@ -90,13 +90,11 @@ type Front struct {
 
 const Sp_state_0, Sp_state_1, Sp_state_2, Sp_state_3, Sp_state_4, Sp_state_5, Sp_state_6, Sp_state_7, Sp_state_8, Sp_state_9, Sp_state_10 = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 //0待完成,1待审核,2打回,3发布,4作废,5已上架,6已下架,7无发布,8需登录,9转python,10已删除
 
-var LuaReg = regexp.MustCompile(`(:(find|match)|string\.|com\.getPureContent)`)
 var spinfos sync.Map = sync.Map{}
 var SessMap map[string]*httpsession.Session
 var AutoTpl map[string]interface{}
 var Mails *util.Mail
 var Reg = regexp.MustCompile(`(http|https)://([\w]+\.)+[\w]+`)
-var ListFilterReg = regexp.MustCompile("关键词过滤")
 var ProjectHrefReg = regexp.MustCompile("projecthref")
 var Transfercode map[string]interface{}
 

+ 73 - 24
src/front/spider.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"log"
 	"mongodb"
-	"regexp"
 	"sort"
 	"spider"
 	"strconv"
@@ -110,7 +109,11 @@ func (f *Front) LoadSpider(codeTaskIdReState string) error {
 		f.T["actiontext"] = "编辑"
 		//lua, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": code})
 		lua, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
-		auth := qu.IntAll(f.GetSession("auth"))
+		//爬虫开发人员编辑爬虫,修改爬虫信息
+		if auth == u.Role_Dev && qu.ObjToString((*lua)["spidertype"]) == "increment" {
+			(*lua)["spidertype"] = "history"
+			(*lua)["incrementevent"] = (*lua)["event"]
+		}
 		if qu.ObjToString((*lua)["createuserid"]) == f.GetSession("userid").(string) || auth >= 1 {
 			if len(*lua) > 0 {
 				if qu.IntAll((*lua)["event"]) == 7000 && qu.IntAll((*lua)["urgency"]) == 0 && qu.IntAll((*lua)["state"]) == 0 {
@@ -174,6 +177,7 @@ func (f *Front) LoadSpider(codeTaskIdReState string) error {
 				f.T["spidermovevent"] = (*lua)["spidermovevent"]
 				f.T["spiderhistorymaxpage"] = (*lua)["spiderhistorymaxpage"]
 				f.T["spiderremark"] = (*lua)["spiderremark"]
+				f.T["identity"] = f.GetSession("identity")
 				events := []string{}
 				for k, _ := range sp.Config.Uploadevents {
 					events = append(events, k)
@@ -186,6 +190,8 @@ func (f *Front) LoadSpider(codeTaskIdReState string) error {
 				f.T["areas"] = u.Area         //
 				f.T["citys"] = u.City         //
 				f.T["provinces"] = u.Province //
+				//查询爬虫是否有列表页处理中的异常任务
+				f.T["clearchannel"] = u.MgoEB.Count("task", map[string]interface{}{"s_code": code, "s_type": "1", "i_state": 2}) > 0
 				return f.Render("spideredit.html", &f.T)
 			}
 		} else {
@@ -434,12 +440,27 @@ func (f *Front) SaveStep() {
 			param["isflow"] = f.OtherBase.IsFlow
 			param["spidertype"] = f.OtherBase.SpiderType
 			param["spiderhistorymaxpage"] = f.OtherBase.SpiderHistoryMaxPage
-			qu.Debug(f.OtherBase.SpiderMoveEvent)
 			tmpEvent, err := strconv.Atoi(f.OtherBase.SpiderMoveEvent) //f.OtherBase.SpiderMoveEvent此处SpiderMoveEvent已不表示comm、bid,表示增量的节点
 			msgResult := map[string]string{}
+			if f.Base.SpiderMaxPage == 1 {
+				msgResult["warn"] = "提醒,增量采集页过小,请再次核对!"
+			}
+			//其他校验
+			if f.Base.SpiderChannel == "" {
+				msgResult["err"] = "栏目名称为空!"
+			}
+			if f.Base.SpiderTargetChannelUrl == "" {
+				msgResult["err"] = "栏目地址为空!;" + msgResult["err"]
+			}
 			if f.OtherBase.SpiderType == "history" { //爬虫类型是history的放到7000节点,并记录历史节点
+				var historyMaxPageWarn string
 				if f.OtherBase.SpiderHistoryMaxPage > 500 { //新爬虫跑历史超过500页提醒新建复制爬虫
-					msgResult["warn"] = "提醒,历史页过大,考虑复制爬虫!"
+					historyMaxPageWarn = "提醒,历史页过大,考虑复制爬虫!"
+				} else if f.OtherBase.SpiderHistoryMaxPage == 1 {
+					historyMaxPageWarn = "提醒,历史页过小,请再次核对!"
+				}
+				if historyMaxPageWarn != "" {
+					msgResult["warn"] = historyMaxPageWarn + ";" + msgResult["warn"]
 				}
 				param["event"] = 7000
 				if err == nil {
@@ -465,7 +486,7 @@ func (f *Front) SaveStep() {
 			param["str_recontent"] = f.StepRe3.Expert
 			param["iscopycontent"] = f.StepRe3.Checked
 			//
-			param["listisfilter"] = ListFilterReg.MatchString(f.Step2.Expert) //列表页校验是否含“--关键词过滤”
+			param["listisfilter"] = u.ListFilterReg.MatchString(f.Step2.Expert) //列表页校验是否含“--关键词过滤”
 			param["projecthref"] = ProjectHrefReg.MatchString(f.Step2.Expert) || ProjectHrefReg.MatchString(f.Step3.Expert)
 			checkLua := LuaTextCheck(infoformat, param, param_list_chrome, param_list_rangechrome, param_content_chrome, msgResult)
 			//checkLua := LuaTextCheck(f.Base.SpiderCode, f.Step2.Expert, f.Step3.Expert, f.Step2.Types, infoformat, model, msgResult)
@@ -479,7 +500,7 @@ func (f *Front) SaveStep() {
 					}
 					Wlog(f.Base.SpiderName, f.Base.SpiderCode, user, f.GetSession("userid").(string), "修改", param)
 					if warn := msgResult["warn"]; warn != "" {
-						rep["msg"] = "保存成功," + msgResult["warn"]
+						rep["msg"] = "保存成功" + msgResult["warn"]
 					}
 					rep["ok"] = true
 				} else {
@@ -524,6 +545,17 @@ func (f *Front) SaveJs() {
 	f.ServeJson(map[string]interface{}{"ok": true})
 }
 
+// LuaCheckSaveLog 保存机检日志
+func LuaCheckSaveLog(lua *map[string]interface{}) {
+	u.MgoEB.Save("luachecklog", map[string]interface{}{
+		"code":       (*lua)["code"],
+		"check":      (*lua)["check"],
+		"checkok":    (*lua)["checkok"],
+		"checktime":  (*lua)["checktime"],
+		"comeintime": time.Now().Unix(),
+	})
+}
+
 func LuaSaveLog(code, user string, data *map[string]interface{}, stype int) {
 	saveOne, _ := u.MgoEB.FindOne("luasavelog", map[string]interface{}{"state": 0, "code": code})
 	if stype == 0 { //保存记录
@@ -643,17 +675,23 @@ func LuaSaveLog(code, user string, data *map[string]interface{}, stype int) {
 	}
 */
 func LuaTextCheck(infoformat int, param map[string]interface{}, param_list_chrome, param_list_rangechrome, param_content_chrome []sp.ChromeActions, msgResult map[string]string) bool {
+	defer qu.Catch()
 	list := qu.ObjToString(param["str_list"])
 	detail := qu.ObjToString(param["str_content"])
 	type_list := qu.IntAll(param["type_list"])
 	type_content := qu.IntAll(param["type_content"])
 	model, _ := param["model"].(map[string]interface{})
-	defer qu.Catch()
-	//1、异常校验
 	var errmsg, warnmsg string
+	if qu.IntAll(param["type_list"]) == 0 {
+		errmsg += "列表页非专家模式;"
+	}
+	if qu.IntAll(param["type_content"]) == 0 {
+		errmsg += "详情页非专家模式;"
+	}
+	//1、异常校验
 	if type_list == 1 {
-		if LuaReg.MatchString(list) || LuaReg.MatchString(detail) {
-			errmsg += "代码中含有lua原生方法;"
+		if u.LuaReg.MatchString(list) {
+			errmsg += "列表页代码中含有lua原生方法;"
 		}
 		if strings.Contains(list, "downloadByChrome") { //chrome下载方法动作参数判断
 			for _, act := range param_list_chrome {
@@ -677,15 +715,12 @@ func LuaTextCheck(infoformat int, param map[string]interface{}, param_list_chrom
 		if strings.Contains(list, "stringFind") && !strings.Contains(list, "--关键词过滤") {
 			errmsg += "列表页代码有过滤方法stringFind但缺少注释:--关键词过滤;"
 		}
-		sln_reg := regexp.MustCompile(`sendListNum\(pageno,list\)`)
-		slnIndexArr := sln_reg.FindAllStringIndex(list, -1)
+		slnIndexArr := u.SendListNumReg.FindAllStringIndex(list, -1)
 		if len(slnIndexArr) == 0 { //列表页专家模式且不含sendListNum
 			errmsg += "代码中缺少sendListNum(pageno,list)方法;"
 		} else if len(slnIndexArr) > 0 { //判断sendListNum方法的位置
-			trim_reg := regexp.MustCompile("trim")
-			insert_reg := regexp.MustCompile("insert")
-			trIndexArr := trim_reg.FindAllStringIndex(list, -1)
-			irIndexArr := insert_reg.FindAllStringIndex(list, -1)
+			trIndexArr := u.TrimReg.FindAllStringIndex(list, -1)
+			irIndexArr := u.InsertReg.FindAllStringIndex(list, -1)
 			slIndex := slnIndexArr[len(slnIndexArr)-1] //sendListNum位置
 			trIndex := trIndexArr[len(trIndexArr)-1]   //com.trim位置
 			irIndex := irIndexArr[len(irIndexArr)-1]   //insert位置
@@ -721,7 +756,9 @@ func LuaTextCheck(infoformat int, param map[string]interface{}, param_list_chrom
 		area := qu.ObjToString(model["area"])
 		city := qu.ObjToString(model["city"])
 		district := qu.ObjToString(model["district"])
-		if area != "" && !strings.Contains(list, area) {
+		if area == "" {
+			errmsg += `模板省份信息不能为空;`
+		} else if !strings.Contains(list, area) {
 			warnmsg += "省份信息与模板不一致;"
 		}
 		if city != "" && !strings.Contains(list, city) {
@@ -760,7 +797,10 @@ func LuaTextCheck(infoformat int, param map[string]interface{}, param_list_chrom
 		}
 	}
 	if type_content == 1 {
-		if ListFilterReg.MatchString(detail) && !strings.Contains(detail, "delete") { //三级页含过滤但是没有data["delete"]="true"
+		if u.LuaReg.MatchString(detail) {
+			errmsg += "详情页代码中含有lua原生方法;"
+		}
+		if u.ListFilterReg.MatchString(detail) && !strings.Contains(detail, "delete") { //三级页含过滤但是没有data["delete"]="true"
 			errmsg += `三级页缺少data["delete"]="true";`
 		}
 		if !strings.Contains(detail, "s_title") {
@@ -779,8 +819,8 @@ func LuaTextCheck(infoformat int, param map[string]interface{}, param_list_chrom
 		warnmsg += "三级页缺少下载附件方法;"
 	}
 	msgResult["warn"] += warnmsg
-	msgResult["err"] = errmsg
-	return errmsg != ""
+	msgResult["err"] += errmsg
+	return msgResult["err"] != ""
 }
 
 // 方法测试
@@ -1080,7 +1120,7 @@ func (f *Front) SpiderPass() {
 	}
 	script, liststr, contentstr := "", "", ""
 	if (*one)["oldlua"] == nil {
-		script, liststr, contentstr = spider.GetScript(code)
+		script, liststr, contentstr = spider.GetScriptByCode(code)
 	} else {
 		script = (*one)["luacontent"].(string)
 	}
@@ -1178,6 +1218,10 @@ func (f *Front) SpiderPass() {
 	steps[6] = s.Test_goreqlist
 	steps[7] = s.Test_goreqcon
 	//校验
+	param_common, _ := (*one)["param_common"].([]interface{})
+	if qu.IntAll(param_common[5]) == 1 && len(result) > 1 { //页码为1,但能翻页
+		msgArr = append(msgArr, "爬虫可以翻页,最大页为1是否合适")
+	}
 	msg := u.SpiderPassCheckListAndDetail(result, data)
 	msgArr = append(msgArr, msg...)
 	username := f.GetSession("username").(string)
@@ -1207,7 +1251,7 @@ func (f *Front) DownSpider(code string) {
 				name := (*one)["createuser"]
 				email := (*user)["s_email"]
 				upload := time.Now().Format("2006-01-02 15:04:05")
-				script, _, _ = spider.GetScript(code, name, email, upload)
+				script, _, _ = spider.GetScriptByCode(code, name, email, upload)
 			}
 		}
 		f.ResponseWriter.Header().Del("Content-Type")
@@ -1563,8 +1607,13 @@ func UpStateAndUpSpider(code, id, reason, username string, state int) (bool, err
 					upset["l_uploadtime"] = time.Now().Unix()
 				} else if state == Sp_state_2 { //打回原因
 					upset["reason"] = reason
-				} else if state == Sp_state_5 { //上架,核对时间重置
-					upset["l_checktime"] = 0
+				} else if state == Sp_state_5 { //上架
+					upset["l_checktime"] = 0 //核对时间重置
+					//清理机检结果
+					unset["check"] = ""
+					unset["checkok"] = ""
+					unset["checktime"] = ""
+					LuaCheckSaveLog(one) //保存机检日志
 					LuaSaveLog(code, username, one, 1)
 					go ModifyLogs_SaveCodeLogs(code, *one)
 				} else if state == Sp_state_4 { //作废,作废原因

+ 1 - 0
src/front/user.go

@@ -125,6 +125,7 @@ func (f *Front) DelUser() {
 			},
 		}
 		ok := u.MgoEB.Update("user", query, update, false, false)
+		u.MgoEB.Del("task", map[string]interface{}{"s_modifyid": userid}) //删除关联任务
 		if ok {
 			f.ServeJson(map[string]interface{}{
 				"status": "y",

+ 508 - 0
src/luacheck/luacheck.go

@@ -0,0 +1,508 @@
+package luacheck
+
+import (
+	"fmt"
+	"github.com/PuerkitoBio/goquery"
+	lua "github.com/yuin/gopher-lua"
+	qu "qfw/util"
+	"regexp"
+	"spider"
+	sp "spiderutil"
+	"strings"
+	"sync"
+	"time"
+	"util"
+)
+
+const SPIDER_MAXPAGENUM = 2 //默认列表页采集页数
+
+var (
+	htmlModelReg    = regexp.MustCompile(`{{[a-zA-z.()\d,:]{5,}}}|^(\$)`) //过滤模板语言
+	reg_filter_url  = regexp.MustCompile(`((\.\./)+|null|[。))]+$)`)
+	reg_invalid_url = regexp.MustCompile(`(^(tel)|^#[\p{Han}]+$|^[\p{Han}]+$|login|mailto|\.(jsp|jspx|aspx|home|com|cn|shtml|jhtml|chtml|html|htm)[))##/、]{0,}$)+`)
+
+	reg_fileter_text = regexp.MustCompile("([<>《》[]()()【】\\[\\]『』。;、;,\\s\u3000\u2003\u00a0]+|(\\\\n)+(\\\\t)+)")
+	reg_filetype     = regexp.MustCompile(`\.(docx|gif|jpg|doc|pdf|rar|png|zip|gz|swf|xlsx|xls|wps|jpeg|javascript:void\(0\)(;))$`)
+	reg_err_filetype = regexp.MustCompile(`(\.(jtbz|jxzf|tytbz|hbz|tbyj|et|tbz|rtf|dwg|bmp|htbz|qttbz|application|zbid|pptx|gef)$|^(#_|file:))`)
+	//全匹配无效内容
+	reg_invalid_text = regexp.MustCompile(`^(\d{4}年\d{1,2}月\d{1,2}日|潜在供应商|递交|查看评论|flash插件|打印文章|收藏|请点击|更多|无|采购详细内容|申请履约保函|关于我们|返回|百度一下|登录(系统)?|查看|网站首页|(免费)?注册|其他|立即报名|我要(报价|投诉|投标|留言)|[\d.])$`)
+	//包含无效关键词
+	reg_filter_text1 = regexp.MustCompile(`(\.(jsp|jspx|aspx|home|com|cn|shtml|jhtml|chtml|html|htm)(/)?$|网站|政府|财产|得分|合同|业绩|负责人|页面|注意事项|注册|投诉|导航|登录|办理|请到原网|我要纠错|([\p{Han}]|\d)+[a-zA-z\d-]{5,}$|[上下首尾](一)?[页篇条]|跳转|详情请见原网站|详见(项目|公告)详情|原(文|公告)链接(地址)?|点击(报名|查看|查阅)(原公告(内容|详情))?|(点[击我])?(查看|查阅)(资质等级树|标的物详情|内容|公告|详情))`)
+	//以关键词结尾
+	reg_filter_text2 = regexp.MustCompile(`((公司|代理|单位|中心|采购办|机构|设计室|(事务|鉴定|研究|管理)所|(卫生|研究|法|设计|医)院|(工程|办事)[部处]|博物馆|工作站|幼儿园|学校|委员会|平台|局|队|[小中大]学)$|(\.{3}|…|管委会|指引|视频|主页|活动|指南|总结|核查|评审|投诉|磋商|调查|列表|处理|须知|审查|名单|需求书|确认书|规则|通知|评价|征询|咨询|采购意向|审计|招标|监理|监测|测量|钻探|测绘|服务|评估|公示|信息|采购|公告|勘察|施工|标段|工程|项目|编制|谈判|意见|设计|邀请函|审核|检测|(意见|建议)书?)$)`)
+
+	CheckStart int
+)
+
+type Data struct {
+	Url        string
+	Text       string
+	Ok         bool
+	By         string
+	FileType   string
+	Base64Type bool
+}
+
+func TimeTaskLuaCheck() {
+	defer qu.Catch()
+	if CheckStart != 0 {
+		return
+	}
+	CheckStart = 1
+	defer func() {
+		CheckStart = 0
+	}()
+	qu.Debug("爬虫质检开始...", time.Now().Unix())
+	query := map[string]interface{}{
+		"event": map[string]interface{}{
+			"$ne": 7410,
+		},
+		"incrementevent": map[string]interface{}{
+			"$ne": 7410,
+		},
+		"platform": "golua平台",
+		"state":    1,
+		"check": map[string]interface{}{
+			"$exists": false,
+		},
+	}
+	list, _ := util.MgoEB.Find("luaconfig", query, nil, nil, false, -1, -1)
+	wg := &sync.WaitGroup{}
+	ch := make(chan bool, 5)
+	for _, l := range *list {
+		ch <- true
+		wg.Add(1)
+		go func(lua map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			warnMap := map[string]string{} //异常集合
+			errMap := map[string]string{}  //错误集合
+			LuaCheck(lua, errMap, warnMap)
+			warnArr := []string{}
+			errArr := []string{}
+			for _, w := range warnMap {
+				warnArr = append(warnArr, w)
+			}
+			for _, e := range errMap {
+				errArr = append(errArr, e)
+			}
+			set := map[string]interface{}{
+				"$set": map[string]interface{}{
+					"check": map[string]interface{}{
+						"warn": warnArr,
+						"err":  errArr,
+					},
+					"checkok":   len(warnArr) == 0 && len(errArr) == 0,
+					"checktime": time.Now().Unix(),
+				},
+			}
+			util.MgoEB.UpdateById("luaconfig", lua["_id"], set)
+		}(l)
+	}
+	wg.Wait()
+	qu.Debug("爬虫质检结束...", time.Now().Unix())
+}
+
+// LuaCheck 爬虫机检
+func LuaCheck(lua map[string]interface{}, err, warn map[string]string) {
+	defer qu.Catch()
+	param_common, _ := lua["param_common"].([]interface{})
+	//1、基本参数
+	//1.1 历史最大页
+	spiderhistorymaxpage := qu.IntAll(lua["spiderhistorymaxpage"])
+	if spiderhistorymaxpage > 500 {
+		warn["历史最大页"] = "历史最大页为" + fmt.Sprint(spiderhistorymaxpage) + ",超过500"
+	} else if spiderhistorymaxpage == 1 {
+		warn["历史最大页"] = "历史最大页为1,过小"
+	}
+	//1.2 最大页
+	pageNum := qu.IntAll(param_common[5])
+	if pageNum == 1 {
+		warn["最大页"] = "最大页为1,过小"
+	}
+	//1.3 省份
+	model, _ := lua["model"].(map[string]interface{})
+	area := qu.ObjToString(model["area"])
+	if area == "" {
+		err["省份"] = "省份参数为空"
+	}
+
+	//2、列表页
+	list := qu.ObjToString(lua["str_list"])
+	//2.1 基本参数是否一致
+	//site、channel、code
+	code := qu.ObjToString(param_common[0])
+	site := qu.ObjToString(param_common[1])
+	channel := qu.ObjToString(param_common[2])
+	if !strings.Contains(list, fmt.Sprintf(util.CheckText_Site, site)) {
+		err["站点模板"] = `站点的值与模板不一致`
+	}
+	if !strings.Contains(list, fmt.Sprintf(util.CheckText_Channel, channel)) {
+		err["栏目模板"] = `栏目的值与模板不一致`
+	}
+	if !strings.Contains(list, fmt.Sprintf(util.CheckText_Code, code)) {
+		err["爬虫代码模板"] = `爬虫代码的值与模板不一致`
+	}
+	//area、city、district
+	city := qu.ObjToString(model["city"])
+	district := qu.ObjToString(model["district"])
+
+	//校验列表页area、city、distric
+	if !strings.Contains(list, util.CheckText_Area) {
+		err["省份模板"] = util.CheckText_Area + "模板不存在"
+	} else if area != "" && !strings.Contains(list, fmt.Sprintf(util.CheckText_Area+`="%s"`, area)) {
+		err["省份模板"] = `省份信息与模板不一致`
+	}
+	if !strings.Contains(list, util.CheckText_City) {
+		err["城市模板"] = util.CheckText_City + "模板不存在"
+	} else if city != "" && !strings.Contains(list, fmt.Sprintf(util.CheckText_City+`="%s"`, city)) {
+		err["城市模板"] = `城市信息与模板不一致`
+	}
+	if !strings.Contains(list, util.CheckText_District) {
+		err["区/县模板"] = util.CheckText_District + "模板不存在"
+	} else if district != "" && !strings.Contains(list, fmt.Sprintf(util.CheckText_District+`="%s"`, district)) {
+		err["区/县模板"] = `区/县信息与模板不一致`
+	}
+	//2.2 sendListNum
+	slnIndexArr := util.SendListNumReg.FindAllStringIndex(list, -1)
+	if len(slnIndexArr) == 0 { //列表页专家模式且不含sendListNum
+		err["sendListNum方法"] = "列表页代码中缺少sendListNum(pageno,list)方法"
+	} else if len(slnIndexArr) > 0 { //判断sendListNum方法的位置
+		trIndexArr := util.TrimReg.FindAllStringIndex(list, -1)
+		irIndexArr := util.InsertReg.FindAllStringIndex(list, -1)
+		slIndex := slnIndexArr[len(slnIndexArr)-1]              //sendListNum位置
+		trIndex := trIndexArr[len(trIndexArr)-1]                //com.trim位置
+		irIndex := irIndexArr[len(irIndexArr)-1]                //insert位置
+		if slIndex[1] < trIndex[0] || slIndex[0] > irIndex[1] { //sendListNum方法必须在com.trim方法后,table.insert方法前
+			err["sendListNum方法"] = "sendListNum方法位置错误"
+		}
+	}
+
+	//2.3 列表页其他参数
+	if qu.IntAll(lua["type_list"]) == 0 { //向导模式
+		err["列表页模式"] = "列表页非专家模式"
+	} else {
+		if text := util.LuaReg.FindString(list); len(text) > 0 {
+			err["列表页原生代码"] = "列表页代码中含有lua原生方法:" + text
+		}
+	}
+
+	//3、详情页
+	detail := qu.ObjToString(lua["str_content"])
+	//3.1 详情页其他参数
+	if qu.IntAll(lua["type_content"]) == 0 { //向导模式
+		err["详情页模式"] = "详情页非专家模式"
+	} else {
+		if util.LuaReg.MatchString(detail) {
+			err["详情页原生代码"] = "详情页代码中含有lua原生方法"
+		}
+		if util.ListFilterReg.MatchString(detail) && !strings.Contains(detail, "delete") { //三级页含过滤但是没有data["delete"]="true"
+			warn["缺少delete代码"] = `详情页代码中缺少data["delete"]="true"`
+		}
+		if !strings.Contains(detail, "s_title") {
+			err["缺少s_title"] = `详情页代码中缺少s_title信息`
+		}
+		//3.2 附件下载
+		if !strings.Contains(detail, "downloadFile") && !strings.Contains(detail, "getFileAttachmentsArrayWithTag") {
+			err["缺少附件下载方法"] = "详情页缺少下载附件方法"
+		}
+	}
+
+	//4、其他参数
+	//5、下载内容
+	luaDownload(pageNum, lua, err, warn)
+}
+
+func luaDownload(pageNum int, lua map[string]interface{}, err, warn map[string]string) {
+	defer qu.Catch()
+	downloadnode := "bid" //使用下载节点test、bid、comm
+	var script string
+	if lua["oldlua"] == nil {
+		script, _, _ = spider.GetScript(lua)
+	} else {
+		script = lua["luacontent"].(string)
+	}
+	s := spider.CreateSpider(downloadnode, script)
+	s.SpiderMaxPage = SPIDER_MAXPAGENUM //采集列表页总页数
+	s.Timeout = 60
+	result := map[int64][]map[string]interface{}{}
+	downloadNum := 0
+	for i := 1; i <= 3; i++ { //列表页重试三次
+		result, downloadNum, _ = s.DownListPageItem() //列表页采集结果
+		if downloadNum > 0 {
+			break
+		}
+	}
+	if downloadNum == 0 {
+		err["列表页下载异常"] = "列表页下载量为0"
+		return
+	}
+	listResultCheck(pageNum, result, err, warn, s) //列表页下载结果校验
+	s.L.Close()
+	//校验
+	//msg := SpiderPassCheckListAndDetail(result, data)
+}
+
+// 列表页下载检测
+func listResultCheck(pageNum int, lists map[int64][]map[string]interface{}, err, warn map[string]string, s *spider.Spider) {
+	defer qu.Catch()
+	//翻页
+	if pageNum > 1 && len(lists) < SPIDER_MAXPAGENUM {
+		warn["列表页翻页异常"] = "列表页翻页失败"
+	} else if pageNum == 1 && len(lists) > 1 {
+		warn["列表页翻页异常"] = "列表页可以翻页,最大页却为1"
+	}
+	var fileNum, downloadOkNum int //附件下载量,详情页下载量
+	for _, list := range lists {
+		for _, l := range list {
+			//校验title
+			title := qu.ObjToString(l["title"])
+			titleRandomArr := util.RandomDFA.CheckSensitiveWord(title) //敏感词匹配乱码集
+			if len(titleRandomArr) > 0 {
+				warn["列表页title含乱码"] = "列表页title含乱码(" + title + "):" + strings.Join(titleRandomArr, "")
+			}
+			if !util.TitleHanReg.MatchString(title) {
+				warn["列表页title无汉字"] = "列表页title中无汉字(" + title + "):"
+			} else if str := util.TitleFilterReg.FindString(title); str != "" {
+				str = "列表页title中包含异常信息(" + title + "):" + str
+				err["列表页title中含异常信息"] = str
+			}
+			//校验发布时间
+			publishtime := qu.ObjToString(l["publishtime"])
+			if publishtime == "0" || publishtime == "" {
+				warn["列表页publishtime"] = "列表页publishtime取值异常(" + title + ")"
+			} else {
+				t, err_p := time.ParseInLocation(qu.Date_Full_Layout, publishtime, time.Local)
+				if err_p != nil || t.Unix() <= 0 || t.Unix() > time.Now().Unix() {
+					warn["列表页publishtime"] = "列表页publishtime取值异常(" + title + ")"
+				}
+			}
+			//href
+			href := qu.ObjToString(l["href"])
+			if str := util.HrefReg.FindString(href); str != "" {
+				err["列表页链接异常"] = "公告链接存在异常后缀(" + title + "):" + str
+			}
+			//详情页下载校验
+			downloadDetail(&fileNum, &downloadOkNum, err, warn, l, s)
+		}
+	}
+	if fileNum == 0 {
+		warn["未下载任何附件"] = "未下载任何附件"
+	}
+	if n := len(lists) - downloadOkNum; n > 0 {
+		warn["部分详情页未下载成功"] = "部分详情页未成功下载,失败" + fmt.Sprint(n) + "条"
+	}
+}
+
+// 详情页下载校验
+func downloadDetail(fileNum, downloadOkNum *int, err, warn map[string]string, list map[string]interface{}, s *spider.Spider) {
+	defer qu.Catch()
+	param := map[string]string{}
+	data := map[string]interface{}{} //详情页下载结果集
+	for k, v := range list {
+		param[k] = qu.ObjToString(v)
+	}
+	var downloadDetailOk bool
+	for i := 1; i <= 3; i++ { //重试三次
+		_, rep_err := s.DownloadDetailPageTest(param, data)
+		if rep_err == nil && len(data) > 0 {
+			downloadDetailOk = true
+			*downloadOkNum++ //记录详情页下载成功个数
+			//详情页基本字段下载结果校验
+			detailResultCheck(data, warn, err)
+			//附件信息校验
+			getFile := false
+			if projectinfo, ok := data["projectinfo"].(map[string]interface{}); ok && projectinfo != nil {
+				if attachments, ok := projectinfo["attachments"].(map[string]interface{}); ok && attachments != nil {
+					for _, tmp := range attachments {
+						tmpMap := tmp.(map[string]interface{})
+						if qu.ObjToString(tmpMap["filename"]) == "附件中含有乱码" {
+							getFile = true
+							warn["附件含乱码"] = "部分附件中含有乱码(" + qu.ObjToString(param["href"]) + ")"
+						} else if qu.ObjToString(tmpMap["org_url"]) != "" && qu.ObjToString(tmpMap["fid"]) == "" { //附件未下载成功
+							warn["部分附件未成功下载"] = "部分附件未成功下载(" + qu.ObjToString(param["href"]) + ")"
+							getFile = true
+						} else if qu.ObjToString(tmpMap["fid"]) != "" {
+							getFile = true
+						}
+					}
+				}
+			}
+			if getFile {
+				*fileNum++ //记录附件下载个数
+			} else {
+				contenthtml := qu.ObjToString(data["contenthtml"])
+				//if !checkFile(contenthtml) {
+				//	err["contenthtml中含有可下载内容"] = "contenthtml中含有可下载内容"
+				//}
+				if text := util.ContentHtmlFilterReg.FindString(contenthtml); text != "" || checkFile(contenthtml) {
+					warn["contenthtml中含有可下载附件"] = "contenthtml中含有可下载附件未下载(" + qu.ObjToString(param["href"]) + ")"
+				}
+			}
+			return
+		}
+	}
+	if !downloadDetailOk {
+		warn["详情页下载失败"] = "详情页下载失败(" + qu.ObjToString(param["href"]) + ")"
+	}
+}
+
+// 详情页下载检测
+func detailResultCheck(data map[string]interface{}, warn, err map[string]string) {
+	defer qu.Catch()
+	//校验s_title
+	if data["s_title"] == nil {
+		err["s_title缺失"] = "详情页s_title缺失"
+	} else {
+		//校验s_title
+		s_title := qu.ObjToString(data["s_title"])
+		s_titleRandomArr := util.RandomDFA.CheckSensitiveWord(s_title) //敏感词匹配乱码集
+		if len(s_titleRandomArr) > 0 {
+			warn["s_title含乱码"] = "s_title含乱码:" + strings.Join(s_titleRandomArr, "")
+		}
+		if !util.TitleHanReg.MatchString(s_title) {
+			warn["s_title无汉字"] = "s_title中无汉字"
+		} else if str := util.TitleFilterReg.FindString(s_title); str != "" {
+			str = "s_title中包含异常信息:" + str
+			err["s_title中含异常信息"] = str
+		}
+	}
+	//校验title
+	title := qu.ObjToString(data["title"])
+	href := qu.ObjToString(data["href"])
+	titleRandomArr := util.RandomDFA.CheckSensitiveWord(title) //敏感词匹配乱码集
+	if len(titleRandomArr) > 0 {
+		warn["详情页title含乱码"] = "详情页title含乱码(" + href + "):" + strings.Join(titleRandomArr, "")
+	}
+	if !util.TitleHanReg.MatchString(title) {
+		warn["详情页title无汉字"] = "详情页title中无汉字(" + href + ")"
+	} else if str := util.TitleFilterReg.FindString(title); str != "" {
+		str = "详情页title中包含异常信息(" + href + "):" + str
+		warn["详情页title中含异常信息"] = str
+	}
+	//校验publishtime
+	if l_np_publishtime, ok := data["l_np_publishtime"].(lua.LNumber); ok {
+		if l_np_publishtime <= 0 || l_np_publishtime > 0 && l_np_publishtime < 1000000000 {
+			warn["详情页l_np_publishtime"] = "详情页l_np_publishtime取值异常(" + href + ")"
+		}
+	} else if l_np_publishtime, ok := data["l_np_publishtime"].(int64); ok {
+		if l_np_publishtime <= 0 || l_np_publishtime > 0 && l_np_publishtime < 1000000000 || l_np_publishtime > time.Now().Unix() {
+			warn["详情页l_np_publishtime"] = "详情页l_np_publishtime取值异常(" + href + ")"
+		}
+	} else {
+		warn["详情页l_np_publishtime"] = "详情页l_np_publishtime取值异常(" + href + ")"
+	}
+	//detail
+	detail := qu.ObjToString(data["detail"])
+	if text := util.DetailFilterReg.FindString(detail); text != "" {
+		err["详情页detail包含无效内容"] = "详情页detail包含无效内容(" + href + "):" + text
+	}
+	detail = sp.Filter.ReplaceAllString(detail, "")              //去空格和html标签
+	detailRandomArr := util.RandomDFA.CheckSensitiveWord(detail) //敏感词匹配乱码集
+	if len(detailRandomArr) > 10 {
+		err["detail含乱码"] = "详情页detail含乱码(" + href + "):" + strings.Join(detailRandomArr, "")
+	}
+	//contenthtml
+	contenthtml := qu.ObjToString(data["contenthtml"])
+	contenthtml = sp.Filter.ReplaceAllString(contenthtml, "")          //去空格和html标签
+	contentRandomArr := util.RandomDFA.CheckSensitiveWord(contenthtml) //敏感词匹配乱码集
+	if len(contentRandomArr) > 10 {
+		err["contenthtml含乱码"] = "详情页contenthtml含乱码(" + href + "):" + strings.Join(contentRandomArr, "")
+	}
+	//校验jsondata
+	if jsondata, ok := data["jsondata"].(map[string]interface{}); ok && len(jsondata) > 0 {
+		for field, _ := range jsondata {
+			if !util.JsonDataMap[field] {
+				err["jsondata属性错误"] = "jsondata中" + field + "属性错误"
+			}
+		}
+	}
+}
+
+func checkFile(contenthtml string) bool {
+	defer qu.Catch()
+	//1、筛选a标签
+	hrefMap := map[string]string{} //记录contenthtml中筛出的a标签信息;key:url,val:text
+	doc, _ := goquery.NewDocumentFromReader(strings.NewReader(contenthtml))
+	doc.Find("a[href]").Each(func(index int, element *goquery.Selection) {
+		attachmentURL, _ := element.Attr("href") //链接
+		if attachmentURL != "" && !htmlModelReg.MatchString(attachmentURL) {
+			hrefMap[attachmentURL] = element.Text()
+		}
+	})
+	result := filterAndDownload(hrefMap) //筛选有效附件链接
+	//if len(result) > 0 {
+	//	_, attachments, attchText := DealAndDownload(result, href) //修复链接和文本并下载附件
+	//}
+	return len(result) > 0
+}
+
+// 筛选有效数据并下载对应附件
+func filterAndDownload(hrefMap map[string]string) (result []*Data) {
+	defer qu.Catch()
+	if len(hrefMap) == 0 {
+		return
+	}
+	for url, text := range hrefMap {
+		//url长度过滤
+		tmpUrl := strings.ToLower(url)
+		if len([]rune(tmpUrl)) <= 10 { //长度
+			continue
+		}
+		result = findFileInfo(tmpUrl, text) //先匹配是否有附件
+		if len(result) > 0 {
+			return
+		}
+		//url无效字符过滤
+		tmpUrl = reg_filter_url.ReplaceAllString(tmpUrl, "")
+		if tmpUrl == "" || reg_invalid_url.MatchString(tmpUrl) {
+			continue
+		}
+		tmpText := strings.ToLower(text)
+		//url、text无效附件类型过滤
+		if reg_err_filetype.MatchString(tmpUrl) || reg_err_filetype.MatchString(tmpText) { //无效附件类型
+			continue
+		}
+		//text过滤
+		tmpText = reg_fileter_text.ReplaceAllString(tmpText, "") //过滤无效字符
+		result = findFileInfo(tmpUrl, text)                      //过滤后找附件
+		if len(result) > 0 {
+			return
+		}
+	}
+	return
+}
+
+func findFileInfo(url, text string) (result []*Data) {
+	defer qu.Catch()
+	if fileType := reg_filetype.FindString(url); fileType != "" { //含常见附件类型结尾的url
+		result = append(result, &Data{
+			Url:      url,
+			Text:     text,
+			By:       "url",
+			FileType: strings.ReplaceAll(fileType, ".", ""),
+		})
+	} else if fileType := reg_filetype.FindString(text); fileType != "" { //含常见附件类型结尾的text
+		result = append(result, &Data{
+			Url:      url,
+			Text:     text,
+			By:       "text",
+			FileType: strings.ReplaceAll(fileType, ".", ""),
+		})
+	} else {
+		//textStr = reg_fileter_text.ReplaceAllString(textStr, "")  //过滤无效字符
+		if reg_invalid_text.ReplaceAllString(text, "") == "" { //无效,全文本匹配,舍弃
+			return
+		} else if reg_filter_text1.MatchString(text) || reg_filter_text2.MatchString(text) { //无效,部分文本匹配,舍弃
+			return
+		}
+		result = append(result, &Data{
+			Url:  url,
+			Text: text,
+			By:   "filter",
+		})
+	}
+	return
+}

+ 1 - 0
src/main.go

@@ -34,6 +34,7 @@ func init() {
 	qu.ReadConfig(&util.Config)
 	qu.ReadConfig("autoimport.json", &front.AutoTpl)
 	qu.ReadConfig("transfercode.json", &front.Transfercode)
+	u.InitRandomCode()
 	//vps
 	qu.ReadConfig("vpsconfig.json", &vps.VpsConfig)
 	//redis

+ 1 - 1
src/main_test.go

@@ -142,7 +142,7 @@ import (
 //	}
 //}
 
-//获取拼音首子母
+// 获取拼音首子母
 func Test_getFirstName(t *testing.T) {
 	str := gopinyin.Convert("HA_河南省政府采购网_招标公告", true)
 	log.Println(str)

Різницю між файлами не показано, бо вона завелика
+ 1 - 0
src/randomcode.json


+ 8 - 6
src/spider/download.go

@@ -1,4 +1,5 @@
-/**
+/*
+*
 GO代码相对简单,
 重点处理下载工具,爬虫启动,监控等。
 逻辑处理交给LUA处理
@@ -21,13 +22,14 @@ import (
 
 var regImgStr = "\\.(JPG|jpg|jpeg|JPEG|GIF|gif|PNG|png|BMP|bmp|doc|docx|pdf|xls|xlsx)$"
 var regImg *regexp.Regexp
-var GarbledCodeReg = regexp.MustCompile("[纰锟绲庯卞鍤滐銇鐟閫嚜鎯壐璩鏉彲鍋撅绺閲嗭絣鐤鏅盫鎽亰寰钂鎳鍒鐏宀婾嗚亗鎬憰攬鍙嶁鑻疐璁鐞鏇顭庮渾寮鑶剸鐙鈪鍐実綍擄鐒鐛绫瀵珐鍡閬栬憟灞綅顡韪忚鍓笉犵鍎鐥慪璜钀氭畯焛鎲顏熺崿鍜鍩僜鍚褰囶鍘櫥闀撹棢檅閯嗏絖灦戝閹涜闇鐮捒鈥璺籏绶澶鎷樺鍌絒嗘鍊ク鐧榦璞嚟鍢鐡瓼屾煢宄鑽畵鎭鈹鑷稛磭鏋孊钄狅絆鐘塋尟鑺絍绂绗嘐幇璨閾戭嚦鐫婅檴碭妤鑴厷挰鐜縒闆憁鏃鐗猒鏁橈顤秨哵鍧紛濊閷顥閺惪鐓嶈亙濠掗帾媞鏀慿瓙鎺闁鎰鑸鎹皝鍔鍦骞閶鍞挾鎴竗閵繉闋戞籅閽欏閼縲鐣呮墔顐ら憼檾锝挻顚炶姂剾鐑鐭潛閰涳楂懘願澧亣倴鐦忕嫄刡灏棙宓媐铇甀鏂楁従態瀹揕闃姒炲矕鏌眱鍍熸腹儝绱獻鐬鑵矦鍝嗗墹崇琛勭仈濴顒剭閴鍏鐝曨锛よ顧勯槈夊潏鐖垚矑鍛瞋終缂鐪鍠鏆妫攏顪娌濆嘇璎厫鍗閮顝給榇婂唭姘燁鏍鑹笎爑嚔槌瀣糵炵櫤鐎闅ゅ類鐨夛绋搕缃娉犲搻鐠儧鋸闉攜楸ㄨ埧欒闊垱鈩厔弐顠拵鑾]+")
+
+//var GarbledCodeReg = regexp.MustCompile("[纰锟绲庯卞鍤滐銇鐟閫嚜鎯壐璩鏉彲鍋撅绺閲嗭絣鐤鏅盫鎽亰寰钂鎳鍒鐏宀婾嗚亗鎬憰攬鍙嶁鑻疐璁鐞鏇顭庮渾寮鑶剸鐙鈪鍐実綍擄鐒鐛绫瀵珐鍡閬栬憟灞綅顡韪忚鍓笉犵鍎鐥慪璜钀氭畯焛鎲顏熺崿鍜鍩僜鍚褰囶鍘櫥闀撹棢檅閯嗏絖灦戝閹涜闇鐮捒鈥璺籏绶澶鎷樺鍌絒嗘鍊ク鐧榦璞嚟鍢鐡瓼屾煢宄鑽畵鎭鈹鑷稛磭鏋孊钄狅絆鐘塋尟鑺絍绂绗嘐幇璨閾戭嚦鐫婅檴碭妤鑴厷挰鐜縒闆憁鏃鐗猒鏁橈顤秨哵鍧紛濊閷顥閺惪鐓嶈亙濠掗帾媞鏀慿瓙鎺闁鎰鑸鎹皝鍔鍦骞閶鍞挾鎴竗閵繉闋戞籅閽欏閼縲鐣呮墔顐ら憼檾锝挻顚炶姂剾鐑鐭潛閰涳楂懘願澧亣倴鐦忕嫄刡灏棙宓媐铇甀鏂楁従態瀹揕闃姒炲矕鏌眱鍍熸腹儝绱獻鐬鑵矦鍝嗗墹崇琛勭仈濴顒剭閴鍏鐝曨锛よ顧勯槈夊潏鐖垚矑鍛瞋終缂鐪鍠鏆妫攏顪娌濆嘇璎厫鍗閮顝給榇婂唭姘燁鏍鑹笎爑嚔槌瀣糵炵櫤鐎闅ゅ類鐨夛绋搕缃娉犲搻鐠儧鋸闉攜楸ㄨ埧欒闊垱鈩厔弐顠拵鑾]+")
 
 func init() {
 	regImg, _ = regexp.Compile(regImgStr)
 }
 
-//下载页面,发送消息,等待别人下载
+// 下载页面,发送消息,等待别人下载
 func Download(downloadnode, downloaderid, url, method string, head map[string]interface{}, encoding string, useproxy, ishttps bool, code string, timeout int64) string {
 	defer mu.Catch()
 	ResultMsclient := MsclientTest
@@ -89,7 +91,7 @@ func Download(downloadnode, downloaderid, url, method string, head map[string]in
 	}
 }
 
-//下载页面,发送消息,等待别人下载
+// 下载页面,发送消息,等待别人下载
 func DownloadAdv(downloadnode, downloaderid, url, method string, reqparam, head map[string]interface{}, mycookie []*http.Cookie, encoding string, useproxy, ishttps bool, code string, timeout int64) (string, []*http.Cookie, map[string]interface{}) {
 	defer mu.Catch()
 	ResultMsclient := MsclientTest
@@ -408,7 +410,7 @@ func DownloadByChrome(downloadnode, downloaderid string, chrometask util.ChromeT
 	return
 }
 
-//下载点是否可用
+// 下载点是否可用
 func isAvailable(code string) bool {
 	b := false
 	for k, _ := range Alldownloader {
@@ -419,7 +421,7 @@ func isAvailable(code string) bool {
 	return b
 }
 
-//下载点是否可用
+// 下载点是否可用
 func isAvailableFile(code string) bool {
 	b := false
 	for k, _ := range AlldownloaderFile {

+ 11 - 5
src/spider/script.go

@@ -30,6 +30,7 @@ import (
 	"strconv"
 	"strings"
 	"time"
+	"util"
 
 	"golang.org/x/text/encoding/simplifiedchinese"
 	"golang.org/x/text/transform"
@@ -370,13 +371,17 @@ func (s *Script) LoadScript(site *string, downloadnode, script string, isfile ..
 			fileName = strings.TrimSpace(fileName)
 			url = strings.TrimSpace(url)
 			ret := DownloadFile(s.Downloader, url, method, sp.GetTable(param), sp.GetTable(head), mycookie, s.Encoding, false, ishttps, "", s.Timeout)
-			qu.Debug(GarbledCodeReg.FindAllString(string(ret), -1), len(ret))
-			if ret == nil || len(ret) < 1024*3 {
+			qu.Debug(util.RandomDFA.CheckSensitiveWord(string(ret)), len(ret))
+			if ret == nil || len(ret) == 0 {
+				name = "附件下载失败!"
 				qu.Debug("下载文件出错!")
+			} else if len(ret) > 0 && len(ret) < 1024*3 {
+				name = "附件过小被过滤!(无效附件)"
 			} else {
 				ftype = qu.GetFileType(ret)
 				url, name, size, ftype, fid = sp.UploadFile(s.SCode, fileName, url, ret)
-				if (ftype == "docx" || ftype == "doc") && len(GarbledCodeReg.FindAllString(string(ret), -1)) > 10 {
+				//if (ftype == "docx" || ftype == "doc") && len(GarbledCodeReg.FindAllString(string(ret), -1)) > 10 {
+				if (ftype == "docx" || ftype == "doc") && len(util.RandomDFA.CheckSensitiveWord(string(ret))) > 10 {
 					name = "附件中含有乱码"
 				}
 			}
@@ -790,12 +795,13 @@ func (s *Script) LoadScript(site *string, downloadnode, script string, isfile ..
 		ret := NewDownloadFile(s.Downloader, url, method, sp.GetTable(param), sp.GetTable(head), mycookie, s.Encoding, false, ishttps, "", s.Timeout, false)
 
 		name, size, ftype, fid := "", "", "", ""
-		qu.Debug(GarbledCodeReg.FindAllString(string(ret), -1), len(ret))
+		qu.Debug(util.RandomDFA.CheckSensitiveWord(string(ret)), len(ret))
 		if ret == nil || len(ret) < 1024*3 {
 			qu.Debug("下载文件出错!")
 		} else {
 			ftype = qu.GetFileType(ret)
-			if (ftype == "docx" || ftype == "doc") && len(GarbledCodeReg.FindAllString(string(ret), -1)) > 10 {
+			//if (ftype == "docx" || ftype == "doc") && len(GarbledCodeReg.FindAllString(string(ret), -1)) > 10 {
+			if (ftype == "docx" || ftype == "doc") && len(util.RandomDFA.CheckSensitiveWord(string(ret))) > 10 {
 				url, name, size, ftype, fid = "附件中含有乱码", "附件中含有乱码", "", "", ""
 			} else {
 				url, name, size, ftype, fid = sp.UploadFile(s.SCode, fileName, url, ret)

+ 88 - 11
src/spider/service.go

@@ -14,8 +14,8 @@ import (
 	u "util"
 )
 
-//获取脚本文件
-func GetScript(code string, str ...interface{}) (script, script_list, script_content string) {
+// 获取脚本文件
+func GetScriptByCode(code string, str ...interface{}) (script, script_list, script_content string) {
 	defer mu.Catch()
 	//script := ""
 	//luaconfig := *mongodb.FindOne("luaconfig", `{"code":"`+code+`"}`)
@@ -97,13 +97,88 @@ func GetScript(code string, str ...interface{}) (script, script_list, script_con
 	return
 }
 
-//保存更新脚本
+func GetScript(lua map[string]interface{}) (script, script_list, script_content string) {
+	defer mu.Catch()
+	//script := ""
+	//luaconfig := *mongodb.FindOne("luaconfig", `{"code":"`+code+`"}`)
+	//qu.Debug(code, "lua---", luaconfig)
+	if lua["listcheck"] == nil {
+		lua["listcheck"] = ""
+	}
+	if lua["contentcheck"] == nil {
+		lua["contentcheck"] = ""
+	}
+	if lua != nil && len(lua) > 0 {
+		common := (lua)["param_common"].([]interface{})
+		if len(common) == 15 {
+			common = append(common, "", "", "")
+		} else {
+			return
+		}
+
+		for k, v := range common {
+			if k == 4 || k == 5 || k == 6 || k == 9 || k == 10 {
+				common[k] = qu.IntAll(v)
+			}
+		}
+		script, _ = GetTmpModel(map[string][]interface{}{"common": common})
+		script_time := ""
+		if qu.IntAll(lua["type_time"]) == 0 {
+			time := lua["param_time"].([]interface{})
+			script_time, _ = GetTmpModel(map[string][]interface{}{
+				"time": time,
+			})
+		} else {
+			script_time = lua["str_time"].(string)
+		}
+		//script_list := "" //列表页
+
+		if qu.IntAll(lua["type_list"]) == 0 {
+			list := lua["param_list"].([]interface{})
+			addrs := strings.Split(list[1].(string), "\n")
+			if len(addrs) > 0 {
+				for k, v := range addrs {
+					addrs[k] = "'" + v + "'"
+				}
+				list[1] = strings.Join(addrs, ",")
+			} else {
+				list[1] = ""
+			}
+			script_list, _ = GetTmpModel(map[string][]interface{}{
+				"list":      list,
+				"listcheck": []interface{}{lua["listcheck"]},
+			})
+		} else {
+			script_list = lua["str_list"].(string)
+		}
+		//script_content := "" //三级页
+		if qu.IntAll(lua["type_content"]) == 0 {
+			content := lua["param_content"].([]interface{})
+			script_content, _ = GetTmpModel(map[string][]interface{}{
+				"content":      content,
+				"contentcheck": []interface{}{lua["contentcheck"]},
+			})
+		} else {
+			script_content = lua["str_content"].(string)
+		}
+		script += fmt.Sprintf(util.Tmp_Other, lua["spidertype"], lua["spiderhistorymaxpage"], lua["spidermovevent"], lua["spidercompete"], lua["infoformat"])
+		script += ` 
+			` + script_time + `
+			` + script_list + `
+			` + script_content
+		script = ReplaceModel(script, common, lua["model"].(map[string]interface{}))
+	}
+	return
+}
+
+// 保存更新脚本
 func SaveSpider(code string, param map[string]interface{}) bool {
 	//return mongodb.Update("luaconfig", bson.M{"code": code}, map[string]interface{}{"$set": param}, true, true)
 	return u.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": param}, true, true)
 }
 
-/*获取最后发布时间
+/*
+获取最后发布时间
 comm	通用参数
 param	向导参数
 proficient  专家脚本
@@ -135,7 +210,8 @@ func GetLastPublishTime(comm, param []interface{}, proficient, downloadnode stri
 	return
 }
 
-/*获取列表信息
+/*
+获取列表信息
 comm		通用参数
 param		向导参数
 model		补充模型
@@ -173,7 +249,8 @@ func GetPageList(comm, param []interface{}, model map[string]interface{}, listch
 	return
 }
 
-/*获取三级页信息
+/*
+获取三级页信息
 comm	通用参数
 param	向导参数
 proficient  专家脚本
@@ -211,7 +288,7 @@ func GetContentInfo(comm, param []interface{}, data map[string]interface{}, cont
 	return
 }
 
-//补充模型
+// 补充模型
 func ReplaceModel(script string, comm []interface{}, model map[string]interface{}) string {
 	//补充通用信息
 	commstr := `item["spidercode"]="` + comm[0].(string) + `";`
@@ -227,7 +304,7 @@ func ReplaceModel(script string, comm []interface{}, model map[string]interface{
 	return script
 }
 
-//创建爬虫
+// 创建爬虫
 func CreateSpider(downloadnode, script string, isfile ...string) *Spider {
 	defer mu.Catch()
 	sp := &Spider{}
@@ -269,7 +346,7 @@ func CreateSpider(downloadnode, script string, isfile ...string) *Spider {
 	return sp
 }
 
-//生成爬虫脚本
+// 生成爬虫脚本
 func GetTmpModel(param map[string][]interface{}) (script string, err interface{}) {
 	qu.Try(func() {
 		if param != nil && param["common"] != nil {
@@ -312,7 +389,7 @@ func GetTmpModel(param map[string][]interface{}) (script string, err interface{}
 	return script, err
 }
 
-//生成文件
+// 生成文件
 func CreateFile(code, script string) (string, error) {
 	filepath := "res/" + time.Now().Format("2006/01/02")
 	err := os.MkdirAll(filepath, 0777)
@@ -322,7 +399,7 @@ func CreateFile(code, script string) (string, error) {
 	return filepath, err
 }
 
-//上传脚本
+// 上传脚本
 func UpdateSpiderByCodeState(code, state string, event int) (bool, error) {
 	msgid := mu.UUID(8)
 	data := map[string]interface{}{}

+ 9 - 6
src/taskManager/taskManager.go

@@ -184,10 +184,13 @@ func (t *TaskM) ManagerTask() {
 			t.Render("task.html", &t.T)
 		} else {
 			ids := []string{}
-			if tmpIds := t.GetSession("ids"); tmpIds != nil {
-				ids = qu.ObjArrToStringArr(tmpIds.([]interface{}))
+			identity := qu.IntAll(t.GetSession("identity"))
+			if identity == 0 { //外包管理员
+				if tmpIds := t.GetSession("ids"); tmpIds != nil {
+					ids = qu.ObjArrToStringArr(tmpIds.([]interface{}))
+				}
 			}
-			query := queryCriteria(userid, platform, taskState, event, stype, ids)
+			query := queryCriteria(userid, platform, taskState, event, stype, identity, ids)
 			if search != "" {
 				query["$or"] = []interface{}{
 					map[string]interface{}{"s_code": map[string]interface{}{"$regex": search}},
@@ -264,7 +267,7 @@ func (t *TaskM) Mytask() {
 		//search := strings.Replace(searchStr, " ", "", -1)
 		search := strings.TrimSpace(searchStr)
 		//查询自己的任务
-		query = queryCriteria(userid, platform, taskState, event, stype, []string{})
+		query = queryCriteria(userid, platform, taskState, event, stype, 1, []string{})
 		if search != "" {
 			query["$or"] = []interface{}{
 				map[string]interface{}{"s_code": map[string]interface{}{"$regex": search}},
@@ -999,11 +1002,11 @@ func updateClose(code string) {
 		fmt.Println("closerate关闭任务:code	", flag)
 	}
 }
-func queryCriteria(userid, platform string, taskState, event, stype int, ids []string) (query map[string]interface{}) {
+func queryCriteria(userid, platform string, taskState, event, stype, identity int, ids []string) (query map[string]interface{}) {
 	query = map[string]interface{}{}
 	if userid != "" && userid != "-1" {
 		query["s_modifyid"] = userid
-	} else {
+	} else if identity == 0 {
 		query["s_modifyid"] = map[string]interface{}{
 			"$in": ids,
 		}

+ 8 - 7
src/timetask/timetask.go

@@ -2,7 +2,9 @@ package timetask
 
 import (
 	"fmt"
+	"github.com/cron"
 	"io/ioutil"
+	"luacheck"
 	"net/http"
 	qu "qfw/util"
 	"spider"
@@ -11,8 +13,6 @@ import (
 	"sync"
 	"time"
 	"util"
-
-	"github.com/cron"
 )
 
 var Mail map[string]interface{}
@@ -22,11 +22,12 @@ func TimeTask() {
 	c := cron.New()
 	c.Start()
 	c.AddFunc("0 20 9 ? * MON-FRI", CheckCreateTask)
-	c.AddFunc("0 30 23 * * *", UpdateSiteInfo)    //定时更新站点信息
-	c.AddFunc("0 0 23 * * *", UpdateCodeHeart)    //定时更新爬虫心跳信息
-	c.AddFunc("0 0 */1 ? * *", CheckLuaMove)      //7000节点转增量爬虫失败告警
-	c.AddFunc("0 */10 * * * *", SpiderMoveEvent)  //7000节点转增量爬虫
-	c.AddFunc("0 0 8 * * *", UpdateImportantCode) //更新重点网站爬虫信息
+	c.AddFunc("0 30 23 * * *", UpdateSiteInfo)            //定时更新站点信息
+	c.AddFunc("0 0 23 * * *", UpdateCodeHeart)            //定时更新爬虫心跳信息
+	c.AddFunc("0 0 */1 ? * *", CheckLuaMove)              //7000节点转增量爬虫失败告警
+	c.AddFunc("0 */10 * * * *", SpiderMoveEvent)          //7000节点转增量爬虫
+	c.AddFunc("0 0 8 * * *", UpdateImportantCode)         //更新重点网站爬虫信息
+	c.AddFunc("0 */5 0 * * *", luacheck.TimeTaskLuaCheck) //更新重点网站爬虫信息
 }
 
 // 检测创建任务失败的爬虫

+ 80 - 0
src/util/dfa.go

@@ -0,0 +1,80 @@
+package util
+
+import (
+	qu "qfw/util"
+	"strings"
+)
+
+var (
+	RandomCode randomCode
+	RandomDFA  *DFA
+)
+
+type randomCode struct {
+	Code string `json:"code"`
+}
+
+// 初始化乱码集
+func InitRandomCode() {
+	qu.ReadConfig("randomcode.json", &RandomCode)
+	randomcodeArr := strings.Split(RandomCode.Code, ",")
+	RandomDFA = &DFA{}
+	RandomDFA.AddWord(randomcodeArr...)
+}
+
+// DFA
+type DFA struct {
+	Link map[string]interface{}
+}
+
+func (d *DFA) AddWord(keys ...string) {
+	d.AddWordAll(true, keys...)
+}
+func (d *DFA) AddWordAll(haskey bool, keys ...string) {
+	if d.Link == nil {
+		d.Link = make(map[string]interface{})
+	}
+	for _, key := range keys {
+		nowMap := &d.Link
+		for i := 0; i < len(key); i++ {
+			kc := key[i : i+1]
+			if v, ok := (*nowMap)[kc]; ok {
+				nowMap, _ = v.(*map[string]interface{})
+			} else {
+				newMap := map[string]interface{}{}
+				newMap["YN"] = "0"
+				(*nowMap)[kc] = &newMap
+				nowMap = &newMap
+			}
+			if i == len(key)-1 {
+				(*nowMap)["YN"] = "1"
+				if haskey {
+					(*nowMap)["K"] = key
+				}
+			}
+		}
+	}
+}
+
+func (d *DFA) CheckSensitiveWord(src string) (res []string) {
+	tmpMap := make(map[string]int)
+	for j := 0; j < len(src); j++ {
+		nowMap := &d.Link
+		for i := j; i < len(src); i++ {
+			word := src[i : i+1]
+			nowMap, _ = (*nowMap)[word].(*map[string]interface{})
+			if nowMap != nil { // 存在,则判断是否为最后一个
+				if "1" == qu.ObjToString((*nowMap)["YN"]) {
+					s := qu.ObjToString((*nowMap)["K"])
+					tmpMap[s]++
+				}
+			} else {
+				break
+			}
+		}
+	}
+	for k, _ := range tmpMap {
+		res = append(res, k)
+	}
+	return res
+}

+ 42 - 26
src/util/util.go

@@ -18,25 +18,29 @@ import (
 const Role_Admin, Role_Examine, Role_Dev = 3, 2, 1 //管理员,审核员,开发员
 var (
 	//MgoE            *mgo.MongodbSim //编辑器87
-	MgoEB             *mgo.MongodbSim //编辑器163
-	MgoS              *mgo.MongodbSim
-	Province          map[string][]string
-	City              map[string][]string
-	DomainNameReg     = regexp.MustCompile(`(http|https)[::]+`)
-	DownLoadReg       = regexp.MustCompile(`download\(.*?\)`)
-	CodeTypeReg       = regexp.MustCompile(`(utf8|utf-8|gbk)`)
-	TitleFilterReg1   = regexp.MustCompile(`[\p{Han}]`)
-	TitleFilterReg2   = regexp.MustCompile(`((上|下)一(页|篇)|阅读次数)`)
-	DetailFilterReg1  = regexp.MustCompile(`((上|下)一(页|篇)|阅读次数|浏览次数|扫一扫|分享|区块链存证)`)
-	Area              []string //省份
-	DomainReg         = regexp.MustCompile(`(?://).+?(?:[::/])`)
-	SymbolReg         = regexp.MustCompile("[,,\\s\u3000\u2003\u00a0]+")
-	ReplaceReg        = regexp.MustCompile(`[]::/]+`)
-	CheckText         = `item["spidercode"]="%s";item["site"]="%s";item["channel"]="%s"`
-	CheckText_Code    = `item["spidercode"]="%s"`
-	CheckText_Site    = `item["site"]="%s"`
-	CheckText_Channel = `item["channel"]="%s"`
-	JsonDataMap       = map[string]bool{ //jsondata
+	MgoEB                *mgo.MongodbSim //编辑器163
+	MgoS                 *mgo.MongodbSim
+	Province             map[string][]string
+	City                 map[string][]string
+	DomainNameReg        = regexp.MustCompile(`(http|https)[::]+`)
+	DownLoadReg          = regexp.MustCompile(`download\(.*?\)`)
+	CodeTypeReg          = regexp.MustCompile(`(utf8|utf-8|gbk)`)
+	TitleHanReg          = regexp.MustCompile(`[\p{Han}]`)
+	TitleFilterReg       = regexp.MustCompile(`((上|下)一(页|篇)|阅读次数|查看详情|转发|<[^>]*?>)|(\.){3,6}`)
+	DetailFilterReg      = regexp.MustCompile(`((上|下)一(页|篇)|阅读次数|浏览次数|扫一扫|分享|区块链存证)`)
+	ContentHtmlFilterReg = regexp.MustCompile(`(iframe|img)`)
+	Area                 []string //省份
+	DomainReg            = regexp.MustCompile(`(?://).+?(?:[::/])`)
+	SymbolReg            = regexp.MustCompile("[,,\\s\u3000\u2003\u00a0]+")
+	ReplaceReg           = regexp.MustCompile(`[]::/]+`)
+	CheckText            = `item["spidercode"]="%s";item["site"]="%s";item["channel"]="%s"`
+	CheckText_Code       = `item["spidercode"]="%s"`
+	CheckText_Site       = `item["site"]="%s"`
+	CheckText_Channel    = `item["channel"]="%s"`
+	CheckText_Area       = `item["area"]`
+	CheckText_City       = `item["city"]`
+	CheckText_District   = `item["district"]`
+	JsonDataMap          = map[string]bool{ //jsondata
 		"extweight":          true,
 		"projecthref":        true,
 		"sourcewebsite":      true,
@@ -92,6 +96,12 @@ var (
 	}
 	Bu = "_bu" //创建采历史爬虫后缀
 
+	HrefReg        = regexp.MustCompile("\\.(json|JPG|jpg|jpeg|JPEG|GIF|gif|PNG|png||doc|docx|pdf|xls|xlsx)$")
+	LuaReg         = regexp.MustCompile(`(:(find|match)|string\.|com\.getPureContent)`)
+	SendListNumReg = regexp.MustCompile(`sendListNum\(pageno,list\)`)
+	TrimReg        = regexp.MustCompile("trim")
+	InsertReg      = regexp.MustCompile("insert")
+	ListFilterReg  = regexp.MustCompile("关键词过滤")
 )
 
 func InitMgo() {
@@ -218,10 +228,11 @@ func SpiderPassCheckListAndDetail(result map[int64][]map[string]interface{}, dat
 		for _, l := range list {
 			//校验title
 			title := qu.ObjToString(l["title"])
-			if !TitleFilterReg1.MatchString(title) {
+			if !TitleHanReg.MatchString(title) {
 				msgMap["列表页title中无汉字"] = true
-			} else if TitleFilterReg2.MatchString(title) {
-				msgMap["列表页title中含有上(下)一页"] = true
+			} else if str := TitleFilterReg.FindString(title); str != "" {
+				str = "列表页title中包含异常信息" + str
+				msgMap[str] = true
 			}
 			//校验发布时间
 			publishtime := qu.ObjToString(l["publishtime"])
@@ -233,6 +244,11 @@ func SpiderPassCheckListAndDetail(result map[int64][]map[string]interface{}, dat
 					msgMap["列表页publishtime取值异常"] = true
 				}
 			}
+			//href
+			href := qu.ObjToString(l["title"])
+			if str := HrefReg.FindString(href); str != "" {
+				msgMap["公告链接存在异常后缀"+str] = true
+			}
 		}
 	}
 	if len(data) > 0 {
@@ -256,7 +272,7 @@ func SpiderPassCheckListAndDetail(result map[int64][]map[string]interface{}, dat
 			msgMap["contenthtml中含有iframe是否下载"] = true
 		}
 		detail := qu.ObjToString(data["detail"])
-		if DetailFilterReg1.MatchString(detail) {
+		if DetailFilterReg.MatchString(detail) {
 			msgMap["三级页正文提取包含无效内容"] = true
 		}
 		//校验jsondata
@@ -284,10 +300,10 @@ func SpiderPassCheckListAndDetail_back(list []map[string]interface{}, data map[s
 		for _, l := range list {
 			//校验title
 			title := qu.ObjToString(l["title"])
-			if !TitleFilterReg1.MatchString(title) && h_flag {
+			if !TitleHanReg.MatchString(title) && h_flag {
 				msg = append(msg, "列表页title中无汉字")
 				h_flag = false
-			} else if TitleFilterReg2.MatchString(title) && n_flag {
+			} else if TitleFilterReg.MatchString(title) && n_flag {
 				msg = append(msg, "列表页title中含有上(下)一页")
 				n_flag = false
 			}
@@ -314,7 +330,7 @@ func SpiderPassCheckListAndDetail_back(list []map[string]interface{}, data map[s
 				msg = append(msg, "contenthtml中含有img是否下载")
 			}
 			detail := qu.ObjToString(data["detail"])
-			if TitleFilterReg2.MatchString(detail) {
+			if TitleFilterReg.MatchString(detail) {
 				msg = append(msg, "三级页正文提取异常")
 			}
 			//校验jsondata

+ 6 - 0
src/web/staticres/js/common.js

@@ -559,6 +559,7 @@ common.spider.upstate=function(code){
 	})
 }
 common.spider.save=function(a,b){
+	setCookieTime(code,"1",24*60*60)//含列表页异常任务爬虫编辑清空栏目名称和栏目地址
 	var base=common.form.serializeArray("baseinfo");
 	if(window.location.pathname=="/center/spider"){
 		$.get("/center/spider/isrepeat?code="+base["Base.SpiderCode"],function(r){
@@ -667,6 +668,11 @@ function setCookie(name,value){
     expdate.setTime(expdate.getTime() + 30 * 60 * 1000);   //时间
     document.cookie = name+"="+value+";expires="+expdate.toGMTString()+";path=/";
 }
+function setCookieTime(name,value,t){
+	var expdate = new Date();   //初始化时间
+	expdate.setTime(expdate.getTime() + t * 1000);   //时间
+	document.cookie = name+"="+value+";expires="+expdate.toGMTString()+";path=/";
+}
 function getCookie(c_name){
 	if (document.cookie.length>0){
 	  c_start=document.cookie.indexOf(c_name + "=")

+ 197 - 116
src/web/templates/index.html

@@ -201,6 +201,7 @@
 					<th>难易度</th>
 					{{end}}
 					<th class="hidden-xs">操作</th>
+					<th>机检结果</th>
                 </tr>
                 </thead>
               </table>
@@ -218,11 +219,10 @@ $(function(){
               "url": "/js/dataTables.chinese.lang"
           },
 		"columnDefs": [
-			{ "orderable": false, "targets": [0,2,4,5,6,7,9,10,11,12] },
+			{ "orderable": false, "targets": [0,2,4,5,6,7,9,10,11,12,13] },
 			//爬虫类型
 			{"targets":[4], createdCell: function (cell, cellData, rowData, rowIndex, colIndex) {
 				{{if gt (session "auth") 2}}
-				var aInput;
 				$(cell).click(function () {
 					$(this).html(updateInfoformat(rowData._id,rowData.state,rowData.code));
 					var aInput = $(this).find(":input");
@@ -237,13 +237,13 @@ $(function(){
 				$(cell).on("blur", ":input", function () {
 					var text = $(this).find("option:selected").text();
 					if (text == "招标"){
-						text = 1
+						text = 1;
 					}else if (text == "拟建/审批"){
-						text = 2
+						text = 2;
 					}else if (text == "产权"){
-						text = 3
+						text = 3;
 					}else if (text == "舆情"){
-						text = 4
+						text = 4;
 					}
 					ttable.cell(cell).data(text);
 					cellData = text;
@@ -253,7 +253,6 @@ $(function(){
 			//更新紧急度
 			{"targets":[5], createdCell: function (cell, cellData, rowData, rowIndex, colIndex) {
 					{{if gt (session "auth") 2}}
-					var aInput;
 					$(cell).click(function () {
 						$(this).html(updateUrgency(rowData._id,rowData.state,rowData.code));
 						var aInput = $(this).find(":input");
@@ -280,10 +279,9 @@ $(function(){
 			//更新节点
 			{"targets":[6], createdCell: function (cell, cellData, rowData, rowIndex, colIndex) {
 				  {{if gt (session "auth") 2}}
-					var aInput;
 					$(cell).click(function () {
 					  $(this).html(createComboxEvent(rowData._id,rowData.state,rowData.code));
-					  var aInput = $(this).find(":input");
+					    var aInput = $(this).find(":input");
 						aInput.focus().val(cellData);
 					});
 					$(cell).on("click", ":input", function (e) {
@@ -298,18 +296,17 @@ $(function(){
 						cellData = text;
 					});
 				{{end}}
-			  }},
+			}},
 			//更新状态
 			{"targets":[9], createdCell: function (cell, cellData, rowData, rowIndex, colIndex) {
 				{{if gt (session "auth") 2}}
-				  var aInput;
 				  $(cell).click(function () {
 					if(cellData!=4&&cellData!=7&&cellData!=8&&cellData!=9&&cellData!=10){
-					  return
+					  return;
 					}
 					$(this).html(createComboxState(rowData._id,rowData.code));
 					var aInput = $(this).find(":input");
-					  aInput.focus().val(cellData);
+					aInput.focus().val(cellData);
 				  });
 				  $(cell).on("click", ":input", function (e) {
 					  e.stopPropagation();
@@ -320,19 +317,19 @@ $(function(){
 				  $(cell).on("blur", ":input", function () {
 					var text = $(this).find("option:selected").text();
 					if(text=="待完成"){
-					  text = 0
+					  text = 0;
 					}else if (text=="已作废"){
-					  text = 4
+					  text = 4;
 					}else if (text=="无发布"){
-					  text = 7
+					  text = 7;
 					}else if (text=="需登录"){
-					  text = 8
+					  text = 8;
 					}else if (text=="转python"){
-					  text = 9
+					  text = 9;
 					}else if (text=="已删除"){
-					  text = 10
+					  text = 10;
 					}else if (text=="已上线"){
-					  text = 11
+					  text = 11;
 					}
 					ttable.cell(cell).data(text);
 					  cellData = text;
@@ -341,10 +338,9 @@ $(function(){
 			  }},
 			//更新平台
 			{"targets":[10], createdCell: function (cell, cellData, rowData, rowIndex, colIndex) {
-				  var aInput;
 				  $(cell).click(function () {
-					$(this).html(createComboxPlatform(rowData.code,rowData._id));
-					var aInput = $(this).find(":input");
+					  $(this).html(createComboxPlatform(rowData.code,rowData._id));
+					  var aInput = $(this).find(":input");
 					  aInput.focus().val(cellData);
 				  });
 				  $(cell).on("click", ":input", function (e) {
@@ -354,14 +350,13 @@ $(function(){
 					  $(this).blur();
 				  });
 				  $(cell).on("blur", ":input", function () {
-					var text = $(this).find("option:selected").text();
+					  var text = $(this).find("option:selected").text();
 					  ttable.cell(cell).data(text);
 					  cellData = text;
 				  });
 			  }},
 			//爬虫挂起
 			{"targets":[11], createdCell: function (cell, cellData, rowData, rowIndex, colIndex) {
-				var aInput;
 				$(cell).click(function () {
 					$(this).html(createComboxPendState(rowData.code));
 					var aInput = $(this).find(":input");
@@ -376,9 +371,9 @@ $(function(){
 				$(cell).on("blur", ":input", function () {
 					var text = $(this).find("option:selected").text();
 					if(text == "否"){
-						text = 0
+						text = 0;
 					}else if(text == "是"){
-						text = 1
+						text = 1;
 					}
 					ttable.cell(cell).data(text);
 					cellData = text;
@@ -405,7 +400,7 @@ $(function(){
 				if(vals.length>15){
 					vals=vals.substr(0,15)+"...";
 				}
-				return  vals
+				return  vals;
 			}},
 			// { "data": "_id","width":"200px",render:function(val,a,row){
 			// 	var text = row["channels"].join("</br>")
@@ -418,123 +413,143 @@ $(function(){
 				if (param_common[11]){
 					href = param_common[11];
 				}
-				return "<a href='"+href+"' target='_blank'>"+val+"</a>"
+				return "<a href='"+href+"' target='_blank'>"+val+"</a>";
 			}},
 			{ "data": "code"},
 		    { "data": "infoformat",render:function(val,a,row){
 				if(val == 1){
-					return "招标"
+					return "招标";
 				}else if (val == 2){
-					return "拟建/审批"
+					return "拟建/审批";
 				}else if (val == 3){
-					return "产权"
+					return "产权";
 				}else if (val == 4){
-					return "舆情"
+					return "舆情";
 				}
 			}},
 			{ "data": "urgency",render:function(val){
 				if(val==1){
-				  return "紧急"
+				  return "紧急";
 				}else{
-				  return "普通"
+				  return "普通";
 				}
 			}},
-			{ "data": "event","width":"50px"},
+			{ "data": "event"},
 			{ "data": "createuser"},
 			{ "data": "modifytime"},
 			{ "data": "state","width":"70px",render:function(val,a,row){
 				var state=row.state;
 				if(state==0){
-					val="<span class='text-info text-bold'>待完成</span>"
+					val="<span class='text-info text-bold'>待完成</span>";
 				}else if(state==1){
-					val="<span class='text-primary text-bold'>待审核</span>"
+					val="<span class='text-primary text-bold'>待审核</span>";
 				}else if(state==2){
-					val="<span class='text-danger text-bold'>未通过</span>"
+					val="<span class='text-danger text-bold'>未通过</span>";
 				}else if(state==3){
-					val="<span class='text-success text-bold'>已通过</span>"
+					val="<span class='text-success text-bold'>已通过</span>";
 				}else if(state==4){
-					val="<span class='text-danger text-bold'>已作废</span>"
+					val="<span class='text-danger text-bold'>已作废</span>";
 				}else if(state==5){
-					val="<span class='text-success text-bold'>已上架</span>"
+					val="<span class='text-success text-bold'>已上架</span>";
 				}else if(state==6){
-					val="<span class='text-danger text-bold'>已下架</span>"
+					val="<span class='text-danger text-bold'>已下架</span>";
 				}else if(state==7){
-					val="<span class='text-danger text-bold'>无发布</span>"
+					val="<span class='text-danger text-bold'>无发布</span>";
 				}else if(state==8){
-					val="<span class='text-danger text-bold'>需登录</span>"
+					val="<span class='text-danger text-bold'>需登录</span>";
 				}else if(state==9){
-					val="<span class='text-danger text-bold'>转python</span>"
+					val="<span class='text-danger text-bold'>转python</span>";
 				}else if(state==10){
-					val="<span class='text-danger text-bold'>已删除</span>"
+					val="<span class='text-danger text-bold'>已删除</span>";
 				}else if(state==11){
-					val="<span class='text-danger text-bold'>已上线</span>"
+					val="<span class='text-danger text-bold'>已上线</span>";
 				}
-				return val
+				return val;
 			}},
 			{ "data": "platform"},
 			{ "data": "pendstate",render:function(val){
 				if(val == 0){
-					return "否"
+					return "否";
 				}else{
-					return "是"
+					return "是";
 				}
 			}},
 			{{if eq (session "platform") "python"}}
 			{ "data": "grade",render:function(val){
 				if(val == undefined||val == 0){
-					return "普通"
+					return "普通";
 				}else if (val == 1){
-					return "较难"
+					return "较难";
 				}else if (val == 2){
-					return "很难"
+					return "很难";
 				}
 			}},
 			{{end}}
 			{ "data": "param_common","width":"95px",render:function(val,a,row){
-				var div=$("<div><div class=\"btn-group\"></div></div>")
-				var currentuser="{{session "username"}}";
+				var div=$("<div><div class=\"btn-group\"></div></div>");
 				var auth="{{session "auth"}}";
 				var state=row.state;
 				{{if eq (session "auth") 1}}
-					var title="查看"
-					var href="/center/spider/view/"+row["encode"]
+					var title="查看";
+					var href="/center/spider/view/"+row["encode"];
+					var button=$('<a type="button" target="_blank" class="btn btn-sm btn-info">'+title+'</a>');
 					if(state==0||state==2){
 						title="编辑"
-						href="/center/spider/edit/"+row["encode"]+"__"
+						href="/center/spider/edit/"+row["encode"]+"__";
+						button=$('<a type="button" target="_blank" class="btn btn-sm btn-primary">'+title+'</a>');
 					}
-					var button=$('<a type="button" target="_blank" class="btn btn-sm btn-default">'+title+'</a>');
+
 					button.attr("href",href);
 					div.find(".btn-group").append(button);
 				{{else if eq (session "auth") 2}}
-					var title="查看"
-					var href="/center/spider/view/"+row["encode"]
+					var title="查看";
+					var href="/center/spider/view/"+row["encode"];
 					if(state==1){
-						title="审核"
-						href="/center/spider/edit/"+row["encode"]+"__"
+						title="审核";
+						href="/center/spider/edit/"+row["encode"]+"__";
 					}
-					var button=$('<a type="button" target="_blank" class="btn btn-sm btn-default">'+title+'</a>');
+					var button=$('<a type="button" target="_blank" class="btn btn-sm btn-primary">'+title+'</a>');
 					button.attr("href",href);
 					div.find(".btn-group").append(button);
 					if(state==1){
-						var dbutton=$('<a type="button" href="#" class="btn btn-sm btn-default" onclick="refuse(this);">打回</a>');
+						var dbutton=$('<a type="button" href="#" class="btn btn-sm btn-danger" onclick="refuse(this);">打回</a>');
 						dbutton.attr("hrefs","/center/spider/upstate?code="+row["code"]+"&state=2");
 						div.find(".btn-group").append(dbutton);
 					}
 				{{else}}
-					var button=$('<a type="button" target="_blank" class="btn btn-sm btn-default">编辑</a>');
+					var button=$('<a type="button" target="_blank" class="btn btn-sm btn-primary">编辑</a>');
 					button.attr("href","/center/spider/edit/"+row["encode"]+"__");
 					div.find(".btn-group").append(button);
 					if(state>2){
-						var button=$('<a type="button" class="btn btn-sm btn-default">下载</a>');
+						var button=$('<a type="button" class="btn btn-sm btn-success">下载</a>');
 						button.attr("href","/center/spider/download/"+row["code"]);
 						div.find(".btn-group").append(button);
 					}else if(state==1) {
-						var dbutton=$('<a type="button" href="#" class="btn btn-sm btn-default" onclick="refuse(this);">打回</a>');
+						var dbutton=$('<a type="button" href="#" class="btn btn-sm btn-danger" onclick="refuse(this);">打回</a>');
 						dbutton.attr("hrefs","/center/spider/upstate?code="+row["code"]+"&state=2");
 						div.find(".btn-group").append(dbutton);
 					}
 				{{end}}
-				return div.html()
+				return div.html();
+			}},
+			{ "data": "param_common","width":"95px",render:function(val,a,row){
+				var div=$("<div><div class=\"btn-group\"></div></div>");
+				if (row.state == 1){
+					var dbutton;
+					if (typeof row.checkok === "undefined"){
+						dbutton = $('<a type="button" class="btn btn-sm disabled btn-default">未机检</a>');
+					}else if(row.checkok){
+						dbutton=$('<a type="button" class="btn btn-sm btn-success">正常</a>');
+					}else{
+						dbutton=$('<a type="button" class="btn btn-sm btn-warning" onclick="showcheck(this);">异常</a>');
+						dbutton.attr("hrefs","/center/spider/upstate?code="+row["code"]+"&state=2");
+					}
+					div.find(".btn-group").append(dbutton);
+				}else{
+					var dbutton=$('<a type="button" class="btn btn-sm disabled btn-warning">无法机检</a>');
+					div.find(".btn-group").append(dbutton);
+				}
+				return div.html();
 			}}
     	],
 		"fnDrawCallback": function(table) {
@@ -548,7 +563,7 @@ $(function(){
 		    });    
 		}, 
 		"fnServerParams": function (e) {  
-			var state=$("#type_state").val()
+			var state=$("#type_state").val();
 			var taskEvent = $("#task_event").val();
 			var urgency = $("#luaurgency").val();
 			var platform = $("#platform").val();
@@ -556,15 +571,15 @@ $(function(){
 			var modifyuser = $("#modifyuser").val();
 			//紧急度
 			if(urgency){
-				e.urgency=urgency
+				e.urgency=urgency;
 			}else{
-				e.urgency="-1"
+				e.urgency="-1";
 			}
 		    //状态
 		    if(state){
-				e.state=state
+				e.state=state;
 			}else{
-				e.state="-1"
+				e.state="-1";
 			}
 			//节点
 			if(taskEvent){
@@ -574,21 +589,21 @@ $(function(){
 			}
 			//平台
 			if(platform){
-				e.platform=platform
+				e.platform=platform;
 			}else{
-				e.platform="-1"
+				e.platform="-1";
 			}
 			//类型
 			if(infoformat){
-				e.infoformat=infoformat
+				e.infoformat=infoformat;
 			}else{
-				e.infoformat="-1"
+				e.infoformat="-1";
 			}
 			//类型
 			if(modifyuser){
-				e.modifyuser=modifyuser
+				e.modifyuser=modifyuser;
 			}else{
-				e.modifyuser="-1"
+				e.modifyuser="-1";
 			}
 		}
 	});
@@ -604,7 +619,7 @@ $(function(){
 			"<option value='-1'>全部</option>"+
       "<option value='0'>普通</option>"+
       "<option value='1'>紧急</option>"+
-			"</select></div>"
+			"</select></div>";
 		$("#spider_filter").prepend("&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;");
 		$("#spider_filter").prepend(urgency);
     	//爬虫状态
@@ -622,28 +637,28 @@ $(function(){
 			"<option value='10'>已删除</option>"+
       "<option value='11'>已上线</option>";
 		{{if eq (session "auth") 3}}
-			opt=opt+opt1+opt2+opt3+opt4
+			opt=opt+opt1+opt2+opt3+opt4;
 		{{else if eq (session "auth") 2}}
-			opt=opt2+opt4
+			opt=opt2+opt3+opt4;
 		{{else}}
-			opt=opt+opt1+opt2+opt3+opt4
+			opt=opt+opt1+opt2+opt3+opt4;
 		{{end}}
 		var select="<div class='form-group'><label for='name'>状态:</label>"+
 			"<select id='type_state' onchange='checkclick(this.value)' class='form-control input-sm'>"+
 			opt+
-			"</select></div>"
+			"</select></div>";
 		$("#spider_filter").prepend("&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;");
 		$("#spider_filter").prepend(select);
 		//节点
-		var spiderEvent="<option value='-1'>全部</option>"
-		var events={{.T.events}}
+		var spiderEvent="<option value='-1'>全部</option>";
+		var events={{.T.events}};
 		for(k in events){
-			spiderEvent+="<option value='"+events[k]+"'>"+events[k]+"</option>"
+			spiderEvent+="<option value='"+events[k]+"'>"+events[k]+"</option>";
 		}
 		var selectEvent="<div class='form-group'><label for='name'>节点:</label>"+
 			"<select id='task_event' onchange='checkclick(this.value)' class='form-control input-sm'>"+
 			spiderEvent+
-			"</select></div>"
+			"</select></div>";
 		$("#spider_filter").prepend("&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;");
 		$("#spider_filter").prepend(selectEvent);
 		//类型
@@ -654,7 +669,7 @@ $(function(){
 				"<option value='2'>拟建/审批</option>"+
 				"<option value='3'>产权</option>"+
 				"<option value='4'>舆情</option>"+
-				"</select></div>"
+				"</select></div>";
 		$("#spider_filter").prepend("&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;");
 		$("#spider_filter").prepend(infoformat);
 		//平台
@@ -665,27 +680,93 @@ $(function(){
 				"<option value='chrome'>chrome</option>"+
 			"<option value='python'>python</option>"+
 				"<option value='通用爬虫'>通用爬虫</option>"+
-			"</select></div>"
+			"</select></div>";
 		$("#spider_filter").prepend("&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;");
 		$("#spider_filter").prepend(platform);
 		//开发
 		{{if gt (session "auth") 2}}
-		var modifyuser="<option value='-1'>全部</option>"
-		var modifyusers={{.T.modifyusers}}
+		var modifyuser="<option value='-1'>全部</option>";
+		var modifyusers={{.T.modifyusers}};
 		for(k in modifyusers){
-			modifyuser+="<option value='"+modifyusers[k]["s_name"]+"'>"+modifyusers[k]["s_name"]+"</option>"
+			modifyuser+="<option value='"+modifyusers[k]["s_name"]+"'>"+modifyusers[k]["s_name"]+"</option>";
 		}
 		var selectModifyuser="<div class='form-group'><label for='name'>维护人:</label>"+
-				"<select id='modifyuser' onchange='checkclick(this.value)' class='form-control input-sm'>"+
-				modifyuser+
-				"</select></div>"
+			"<select id='modifyuser' onchange='checkclick(this.value)' class='form-control input-sm'>"+
+			modifyuser+
+			"</select></div>";
 		$("#spider_filter").prepend("&nbsp;&nbsp;");
 		$("#spider_filter").prepend(selectModifyuser);
 		{{end}}
-    	$("#spider_wrapper .col-sm-6").css({width:"100%"})//样式
+    	$("#spider_wrapper .col-sm-6").css({width:"100%"});//样式
 	});
 })
 
+	function showcheck(e){
+		var tr = $(e).closest('tr');
+		var row = ttable.row(tr);
+		if (row.child.isShown()) {
+			// This row is already open - close it
+			row.child.hide();
+			tr.removeClass('shown');
+		} else {
+			// Open this row
+			row.child(format_X(row.data())).show();
+			tr.addClass('shown');
+		}
+	}
+
+	function format_X(d) {
+	var data = Object.assign({},d);
+	data = {...d};
+	var errInfo = data.check.err;
+	var errLen = Object.keys(errInfo).length;
+	var warnInfo = data.check.warn;
+	var warnLen = Object.keys(warnInfo).length;
+
+	var maxLen = Math.max(errLen, warnLen);
+	var errth = '';
+	var warnth = '';
+	if (errLen > 0){
+		errth = '<th>错误信息</th>';
+	}
+	if (warnLen > 0){
+		warnth = '<th>警告信息</th>';
+	}
+	var checkHtml = '';
+	for (let i = 0; i < maxLen; i++) {
+		var ei = errInfo[i];
+		var wi = warnInfo[i];
+		if (!ei){
+			ei = '';
+		}else{
+			ei = (i+1)+"."+ei;
+		}
+		if (!wi){
+			wi = '';
+		}else{
+			wi = (i+1)+"."+wi;
+		}
+		var errtd = ''
+		var warntd = ''
+		if (errLen > 0){
+			errtd = '<td>' +ei+'</td>';
+		}
+		if (warnLen > 0){
+			warntd = '<td>' +wi+'</td>';
+		}
+		checkHtml += '<tr>' + errtd + warntd + '</tr>';
+	}
+	return (
+		'<table id="nestable" style="border-spacing: 9px;border-collapse: separate;background: aliceblue;">' +
+			'<tr>' +
+				errth +
+				warnth +
+			'</tr>' +
+			checkHtml+
+		'</table>'
+	);
+}
+
 	function checkclick(me){
 		ttable.ajax.reload();
 	}
@@ -724,14 +805,14 @@ $(function(){
 			success: function(r){
 			  if(r&&r.success){
 				$("#modal-spidercopy").modal("hide");
-				window.open("/center/spider/edit/"+r.encode+"__")
+				window.open("/center/spider/edit/"+r.encode+"__");
 			  }else{
-				alert(r.text)
+				alert(r.text);
 			  }
 			}
 		  })
 		}else{
-		  alert("爬虫不能为空!")
+		  alert("爬虫不能为空!");
 		}
     }
 	//拆分复制爬虫
@@ -748,19 +829,19 @@ $(function(){
 						// $("#modal-spidercopysplit").modal("hide");
 						window.location.reload();
 					}else{
-						alert(r.text)
+						alert(r.text);
 					}
 				}
 			})
 		}else{
-			alert("爬虫、数量不能为空!")
+			alert("爬虫、数量不能为空!");
 		}
 	}
   	function createComboxPendState(code){
 		return "<select onchange='changeCodePendState(this.value,\""+code+"\")' class='form-control input-sm'>"+
 			"<option value=1>是</option>"+
 			"<option value=0>否</option>"+
-			"</select>"
+			"</select>";
 	};
 	function changeCodePendState(pendstate,code){
 		$.ajax({
@@ -785,7 +866,7 @@ $(function(){
 				"<option value='chrome'>chrome</option>"+
 				"<option value='python'>python</option>"+
 				"<option value='通用爬虫'>通用爬虫</option>"+
-				"</select>"
+				"</select>";
 	};
 	function updateesp(val,w,s,c,id){
 	$.ajax({
@@ -804,12 +885,12 @@ $(function(){
 			}
 			ttable.ajax.reload();
 		}
-	})
+	});
 }
 
 	{{if gt (session "auth") 1}}
 	function refuse(me){
-		var $r=window.prompt("确定打回?请输入打回原因")
+		var $r=window.prompt("确定打回?请输入打回原因");
 		if($r){
 			$.ajax({
 				url:$(me).attr("hrefs"),
@@ -883,8 +964,8 @@ $(function(){
 	function comfirm_disables(){
 		var state = $("#code-state").val();
 		if (state == -1){
-			alert("请选择爬虫状态!")
-			return
+			alert("请选择爬虫状态!");
+			return;
 		}
 		$("#modal-disables").modal("hide");
 		var events=[];
@@ -993,21 +1074,21 @@ $(function(){
 	//修改爬虫类型
 	function updateInfoformat(id,state,code){
 		var spiderInforformat="<option value=1>招标</option><option value=2>拟建/审批</option><option value=3>产权</option><option value=4>舆情</option>";
-		return "<select onchange='updateesp(this.value,\"infoformat\",\""+state+"\",\""+code+"\",\""+id+"\")' class='form-control input-sm'>"+spiderInforformat+"</select>"
+		return "<select onchange='updateesp(this.value,\"infoformat\",\""+state+"\",\""+code+"\",\""+id+"\")' class='form-control input-sm'>"+spiderInforformat+"</select>";
 	};
 	//修改紧急度
 	function updateUrgency(id,state,code){
 		var spiderUrgency="<option value=0>普通</option><option value=1>紧急</option>";
-		return "<select onchange='updateesp(this.value,\"urgency\",\""+state+"\",\""+code+"\",\""+id+"\")' class='form-control input-sm'>"+spiderUrgency+"</select>"
+		return "<select onchange='updateesp(this.value,\"urgency\",\""+state+"\",\""+code+"\",\""+id+"\")' class='form-control input-sm'>"+spiderUrgency+"</select>";
 	};
     //修改节点
     function createComboxEvent(id,state,code){
     	var events={{.T.events}};
     	var spiderEvent="";
 		for(k in events){
-			spiderEvent+="<option value='"+events[k]+"'>"+events[k]+"</option>"
+			spiderEvent+="<option value='"+events[k]+"'>"+events[k]+"</option>";
 		}
-    	return "<select onchange='updateesp(this.value,\"event\",\""+state+"\",\""+code+"\",\""+id+"\")' class='form-control input-sm'>"+spiderEvent+"</select>"
+    	return "<select onchange='updateesp(this.value,\"event\",\""+state+"\",\""+code+"\",\""+id+"\")' class='form-control input-sm'>"+spiderEvent+"</select>";
     };
     function createComboxState(id,code){
     	return "<select id='task_state'  onchange='updateesp(this.value,\"state\",\"\",\""+code+"\",\""+id+"\")' class='form-control input-sm'>"+
@@ -1023,8 +1104,8 @@ $(function(){
       		"<option disabled value='9'>转python</option>"+
       		"<option disabled value='10'>已删除</option>"+
             "<option disabled value='10'>已上线</option>"+
-          +"</select>"
-  };
+          +"</select>";
+  	};
 {{end}}
 </script>
 </div>

+ 34 - 9
src/web/templates/lualist.html

@@ -170,16 +170,21 @@
                                 var dbutton=$('<button class="btn btn-sm btn-warning" onclick="myconfirm(this,\'上架\');">上架</button>');
                                 dbutton.attr("hrefs","/center/spider/upstate?code="+row["code"]+"&state=5");
                                 div.find(".btn-group").append(dbutton);
+                                if (val==3){
+                                    var rfbutton=$('<button class="btn btn-sm btn-danger" onclick="refuse(this);">打回</button>');
+                                    rfbutton.attr("hrefs","/center/spider/upstate?code="+row["code"]+"&state=2");
+                                    div.find(".btn-group").append(rfbutton);
+                                }
                             }
-                            if(row["is_check"]){
-                                var dbutton1=$('<button class="btn btn-sm btn-success" onclick="common.spider.checkTime(\''+row.code+'\')">核对</button>');
-                                dbutton1.attr("hrefs","");
-                                div.find(".btn-group").append(dbutton1);
-                            }else{
-                                var dbutton1=$('<button class="btn btn-sm btn-danger" onclick="common.spider.checkTime(\''+row.code+'\')">核对</button>');
-                                dbutton1.attr("hrefs","");
-                                div.find(".btn-group").append(dbutton1);
-                            }
+                            // if(row["is_check"]){
+                            //     var dbutton1=$('<button class="btn btn-sm btn-success" onclick="common.spider.checkTime(\''+row.code+'\')">核对</button>');
+                            //     dbutton1.attr("hrefs","");
+                            //     div.find(".btn-group").append(dbutton1);
+                            // }else{
+                            //     var dbutton1=$('<button class="btn btn-sm btn-danger" onclick="common.spider.checkTime(\''+row.code+'\')">核对</button>');
+                            //     dbutton1.attr("hrefs","");
+                            //     div.find(".btn-group").append(dbutton1);
+                            // }
                             var createTask=$('<a type="button" class="btn btn-sm btn-primary">新建</a>');
                             createTask.attr("onclick","createTask('"+row['code']+"')");
                             div.find(".btn-group").append(createTask);
@@ -257,6 +262,26 @@
                 $("#spider_wrapper .col-sm-6").css({width:"100%"})//样式
             });
         })
+
+        function refuse(me){
+            var $r=window.prompt("确定打回?请输入打回原因")
+            if($r){
+                $.ajax({
+                    url:$(me).attr("hrefs"),
+                    type:"post",
+                    data:{"reason":$r},
+                    success:function(r){
+                        if(r["err"] == ""){
+                            showTip("打回成功", 500, function() {});
+                            ttable.ajax.reload(null,false);
+                        }else{
+                            showMsg("打回失败", function() {});
+                        }
+                    }
+                })
+            }
+        }
+
         function changeEvent(code){
             var event=window.prompt("==============请输入更新后的节点==============\n","");
             if(event && event!=""){

+ 1 - 1
src/web/templates/mytask.html

@@ -134,7 +134,7 @@
 					}},
 				{"data":"_id",render:function(val,a,row){
 					var div=$("<div><div class=\"btn-group\"></div></div>")
-					var dbutton=$('<a type="button" href="#" class="btn btn-sm btn-default" onclick="viewtask(\''+row._id+'\',\''+row.relatecode+'\');">查看</a>');
+					var dbutton=$('<a type="button" href="#" class="btn btn-sm btn-info" onclick="viewtask(\''+row._id+'\',\''+row.relatecode+'\');">查看</a>');
 					div.find(".btn-group").append(dbutton);
 					return div.html();
 					// var div=$("<div><div class=\"btn-group\"></div></div>")

+ 28 - 11
src/web/templates/spiderbase.html

@@ -149,6 +149,14 @@
 						placeholder="爬虫栏目" value="{{if .T.lua.param_common}}{{index .T.lua.param_common 2}}{{end}}">
 					</div>
 				</div>
+				<div class="form-group" style="display: none" id="old_spiderChannel">
+					<label for="spiderChannel" class="col-sm-3 control-label">
+						旧栏目
+					</label>
+					<div class="col-sm-8">
+						<input type="text" disabled class="form-control" placeholder="爬虫栏目" value="{{if .T.lua.param_common}}{{index .T.lua.param_common 2}}{{end}}">
+					</div>
+				</div>
         		<div class="form-group" style="display: none">
 					<label for="isflow" class="col-sm-6 control-label text-right">
 						是否是流程爬虫 
@@ -271,7 +279,7 @@
 							爬虫类型
 						</label>
 						<div class="col-sm-7">
-							<select name="OtherBase.SpiderType" id="spiderType" class="form-control" onchange="spiderTypeChange()">
+							<select name="OtherBase.SpiderType" disabled id="spiderType" class="form-control" onchange="spiderTypeChange()">
 								<option value="increment">
 									增量
 								</option>
@@ -419,8 +427,16 @@
 						栏目地址
 					</label>
 					<div class="col-sm-7">
-						<input type="text" class="form-control" name="Base.SpiderTargetChannelUrl"
-						id="spiderTargetChannelUrl" placeholder="http://xxxx" value="{{if .T.lua.param_common}}{{index .T.lua.param_common 11}}{{end}}"
+						<input type="text" class="form-control" name="Base.SpiderTargetChannelUrl" id="spiderTargetChannelUrl" placeholder="http://xxxx" value="{{if .T.lua.param_common}}{{index .T.lua.param_common 11}}{{end}}"
+						/>
+					</div>
+				</div>
+				<div class="form-group" style="display: none" id="old_spiderTargetChannelUrl">
+					<label for="spiderTargetChannelUrl" class="col-sm-4 control-label ">
+						旧栏目地址
+					</label>
+					<div class="col-sm-7">
+						<input type="text" disabled class="form-control" placeholder="http://xxxx" value="{{if .T.lua.param_common}}{{index .T.lua.param_common 11}}{{end}}"
 						/>
 					</div>
 				</div>
@@ -433,19 +449,19 @@
 						<span id="popoverid"></span>
 					</div>
 					<script>
-						$(document).ready(function() {
-							$("body").on("click", function(event) {
-								if (!$(event.target).is("#spiderIsHistoricalMend")){
-									$("#popoverid").css("display","none");
-								}
-							});
-						});
+						// $(document).ready(function() {
+						// 	$("body").on("click", function(event) {
+						// 		if (!$(event.target).is("#spiderIsHistoricalMend")){
+						// 			 $("#popoverid").css("display","none");
+						// 		}
+						// 	});
+						// });
 						if ($("#spiderIsHistoricalMend").val() == "true") {
 						    $("#spiderIsHistoricalMend").attr("checked", true);
 						}else{
 							 $("#spiderIsHistoricalMend").removeAttr("checked");
 						}
-						if ({{.T.spidertype}} != "increment") {
+						if (({{.T.spidertype}} != "increment") || ({{.T.identity}} == 0 && /\_bu$/.test({{.T.lua.code}}))) {
 							$("#spiderIsHistoricalMend").attr("disabled", true);
 						}
 						$("#spiderIsHistoricalMend").click(function() {
@@ -456,6 +472,7 @@
 						    } else {
 								$("#popoverid").text("未选中:爬虫采集增量数据!");
 								// $("#popoverid").css("display","none");
+								$("#popoverid").css("display","inline-block") ;
 						        $(this).val("false")
 						    }
 						});

+ 9 - 0
src/web/templates/spideredit.html

@@ -264,7 +264,16 @@
 	var luaIsChrome = {{.T.lua.platform}} == "chrome";
 	var areas = {{.T.provinces}};
 	var citys = {{.T.citys}};
+	var clearchannel = {{.T.clearchannel}};
+	var code = {{.T.lua.code}};
 	$(function(){
+		//含列表页异常任务爬虫编辑清空栏目名称和栏目地址
+		if(clearchannel && !getCookie(code)){
+			$("#spiderChannel").val("");
+			$("#spiderTargetChannelUrl").val("");
+			$("#old_spiderChannel").css("display","block");
+			$("#old_spiderTargetChannelUrl").css("display","block");
+		}
 		//采集说明
 		if ({{.T.spiderremark}} && {{.T.spiderremark}} != ""){
 			$("#spiderremark").text("采集说明:"+{{.T.spiderremark}});

+ 1 - 0
src/web/templates/spiderview.html

@@ -68,6 +68,7 @@
 	var luaIsChrome = {{.T.lua.platform}} == "chrome";
 	var areas = {{.T.provinces}};
 	var citys = {{.T.citys}};
+	var code = {{.T.lua.code}};
 	$(function(){
 		//采集说明
 		if ({{.T.spiderremark}} && {{.T.spiderremark}} != ""){

+ 4 - 4
src/web/templates/task.html

@@ -221,9 +221,9 @@
 				}},
 				{"data": "_id","width":"10%",render:function(val,a,row){
 					var div=$("<div><div class=\"btn-group\"></div></div>")
-					var buttonWatch=$('<a type="button" target="_blank" class="btn btn-sm btn-success">编辑</a>');
+					var buttonWatch=$('<a type="button" target="_blank" class="btn btn-sm btn-primary">编辑</a>');
 					var buttonClose=$('<a type="button" class="btn btn-sm btn-danger">关闭</a>');
-					var buttonTag=$('<a type="button" class="btn btn-sm btn-primary">标记</a>');
+					var buttonTag=$('<a type="button" class="btn btn-sm btn-warning">标记</a>');
 					//var buttonDel=$('<a type="button" class="btn btn-sm btn-default">删除</a>');
 					buttonWatch.attr("href","/center/task/edit/"+row['_id']+"__0");
 					buttonClose.attr("onclick","closeTask('"+row['s_code']+"','"+row['i_state']+"','"+row['_id']+"')");
@@ -231,7 +231,7 @@
 					div.find(".btn-group").append(buttonWatch);
 				    div.find(".btn-group").append(buttonClose);
 
-					var buttonAssign=$('<a type="button" class="btn btn-sm btn-warning">分发</a>');
+					var buttonAssign=$('<a type="button" class="btn btn-sm btn-info">分发</a>');
 					buttonAssign.attr("onclick","assignTask('"+row['s_code']+"','"+row['i_state']+"','"+row['_id']+"')");
 					if(row['i_state'] == "0"){
 						div.find(".btn-group").append(buttonAssign);
@@ -247,7 +247,7 @@
 			 	$('#dataTable-btn').click(function(e) {    
 				    var redirectpage=0
 					if($("#changePage").val() && $("#changePage").val() > 0) {    
-				        var redirectpage = $("#changePage").val() - 1;    
+				         redirectpage = $("#changePage").val() - 1;
 				    }  
 					ttabletask.page(redirectpage).draw(false);      
 			    });    

Деякі файли не було показано, через те що забагато файлів було змінено