|
@@ -538,17 +538,22 @@ func LuaTextCheck(list, detail string, type_list int) (b bool, msg string) {
|
|
|
b = true
|
|
|
msg = `三级页缺少data["delete"]="true"`
|
|
|
}
|
|
|
- reg1 := regexp.MustCompile("sendListNum")
|
|
|
- slIndexArr := reg1.FindAllStringIndex(list, -1)
|
|
|
- if type_list != 0 && len(slIndexArr) == 0 { //列表页专家模式且不含sendListNum
|
|
|
+ sln_reg := regexp.MustCompile(`sendListNum\(k,list\)`)
|
|
|
+ slnIndexArr := sln_reg.FindAllStringIndex(list, -1)
|
|
|
+ if type_list != 0 && len(slnIndexArr) == 0 { //列表页专家模式且不含sendListNum
|
|
|
b = true
|
|
|
- msg = "代码中缺少sendListNum方法;" + msg
|
|
|
- } else if type_list == 1 && len(slIndexArr) > 0 { //判断sendListNum方法的位置
|
|
|
- reg2 := regexp.MustCompile("insert")
|
|
|
- tsIndexArr := reg2.FindAllStringIndex(list, -1)
|
|
|
- slIndex := slIndexArr[len(slIndexArr)-1]
|
|
|
- tsIndex := tsIndexArr[len(tsIndexArr)-1]
|
|
|
- if slIndex[1] < tsIndex[1] { //sendListNum方法必须在table.inset方法后
|
|
|
+ msg = "代码中缺少sendListNum(k,list)方法;" + msg
|
|
|
+ } else if type_list == 1 && len(slnIndexArr) > 0 { //判断sendListNum方法的位置
|
|
|
+ trim_reg := regexp.MustCompile("trim")
|
|
|
+ insert_reg := regexp.MustCompile("insert")
|
|
|
+
|
|
|
+ trIndexArr := trim_reg.FindAllStringIndex(list, -1)
|
|
|
+ irIndexArr := insert_reg.FindAllStringIndex(list, -1)
|
|
|
+ slIndex := slnIndexArr[len(slnIndexArr)-1] //sendListNum位置
|
|
|
+ trIndex := trIndexArr[len(trIndexArr)-1] //com.trim位置
|
|
|
+ irIndex := irIndexArr[len(irIndexArr)-1] //insert位置
|
|
|
+ qu.Debug("sendListNum位置:", trIndex, slIndex, irIndex)
|
|
|
+ if slIndex[1] < trIndex[0] || slIndex[0] > irIndex[1] { //sendListNum方法必须在com.trim方法后,table.insert方法前
|
|
|
b = true
|
|
|
msg = "sendListNum方法位置错误;" + msg
|
|
|
}
|
|
@@ -689,16 +694,28 @@ func (f *Front) GetJson() {
|
|
|
tr.task_remark = "审核任务创建时间:" + qu.FormatDateByInt64(&comeintime, qu.Date_Short_Layout) + ";" + tr.task_remark
|
|
|
}
|
|
|
data := tr.dataInfo
|
|
|
- list := tr.listInfo
|
|
|
+ result := tr.listInfo
|
|
|
if len(data) > 0 {
|
|
|
data["contenthtml"] = ""
|
|
|
}
|
|
|
- for k, v := range list {
|
|
|
- v["a_index"] = k + 1
|
|
|
+ num := 0
|
|
|
+ list_fir := []map[string]interface{}{}
|
|
|
+ list_sec := []map[string]interface{}{}
|
|
|
+ for page, list := range result {
|
|
|
+ for k, v := range list {
|
|
|
+ v["a_index"] = k + 1
|
|
|
+ num++
|
|
|
+ }
|
|
|
+ if page == 1 {
|
|
|
+ list_fir = list
|
|
|
+ } else if page == 2 {
|
|
|
+ list_sec = list
|
|
|
+ }
|
|
|
}
|
|
|
-
|
|
|
- f.T["list"] = list
|
|
|
+ f.T["list_fir"] = list_fir
|
|
|
+ f.T["list_sec"] = list_sec
|
|
|
f.T["data"] = data
|
|
|
+ f.T["num"] = num
|
|
|
f.T["descript"] = tr.task_descript
|
|
|
f.T["remark"] = tr.task_remark
|
|
|
f.T["rateremark"] = tr.task_rateremark
|
|
@@ -707,7 +724,6 @@ func (f *Front) GetJson() {
|
|
|
f.T["comeintime"] = comeintime
|
|
|
delete(TestResultMap, username+code)
|
|
|
}
|
|
|
-
|
|
|
f.Render("jsonInfo.html", &f.T)
|
|
|
}
|
|
|
|
|
@@ -720,7 +736,7 @@ type TestResult struct {
|
|
|
task_descript string
|
|
|
reason string
|
|
|
msg string
|
|
|
- listInfo []map[string]interface{}
|
|
|
+ listInfo map[int64][]map[string]interface{}
|
|
|
dataInfo map[string]interface{}
|
|
|
}
|
|
|
|
|
@@ -728,7 +744,7 @@ type TestResult struct {
|
|
|
func (f *Front) SpiderPass() {
|
|
|
defer mu.Catch()
|
|
|
tr := &TestResult{}
|
|
|
- list := []map[string]interface{}{}
|
|
|
+ result := map[int64][]map[string]interface{}{}
|
|
|
data := map[string]interface{}{}
|
|
|
msgArr := []string{}
|
|
|
code := f.GetString("code")
|
|
@@ -765,7 +781,7 @@ func (f *Front) SpiderPass() {
|
|
|
tr.task_rateremark = rateremarktmp
|
|
|
tr.task_descript = descript
|
|
|
|
|
|
- //基本信息、方法一、方法二、方法三、总请求次数、go方法一、go方法二、go方法三、列表页条数
|
|
|
+ //基本信息、方法一(发布时间)、方法二(列表页)、方法三(详情页)、总请求次数、go方法一、go方法二、go方法三、列表页条数
|
|
|
steps := []interface{}{false, false, false, false, 0, 0, 0, 0, 0}
|
|
|
one, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
|
|
|
reason, _ := (*one)["reason"].(string)
|
|
@@ -791,59 +807,88 @@ func (f *Front) SpiderPass() {
|
|
|
msgArr = u.SpiderPassCheckLua(liststr, contentstr, (*one)) //校验
|
|
|
}
|
|
|
s := spider.CreateSpider(downloadnode, script)
|
|
|
- s.SpiderMaxPage = 1
|
|
|
+ s.SpiderMaxPage = 2 //采集列表页总页数
|
|
|
s.Timeout = 60
|
|
|
timestr, timeerr := s.GetLastPublishTime()
|
|
|
if timeerr == nil && len(timestr) > 4 {
|
|
|
- steps[1] = true
|
|
|
- list, _ = s.DownListPageItem()
|
|
|
- for _, l := range list {
|
|
|
- if publishtime := qu.ObjToString(l["publishtime"]); publishtime == "0" || publishtime == "" {
|
|
|
- msgArr = append(msgArr, "列表页publishtime取值异常")
|
|
|
- break
|
|
|
- } else {
|
|
|
- t, err := time.ParseInLocation(qu.Date_Full_Layout, publishtime, time.Local)
|
|
|
- if err != nil || t.Unix() <= 0 {
|
|
|
- msgArr = append(msgArr, "列表页publishtime取值异常")
|
|
|
- break
|
|
|
- }
|
|
|
- }
|
|
|
+ steps[1] = true //发布时间获取成功
|
|
|
+ downloadNum := 0
|
|
|
+ result, downloadNum, _ = s.DownListPageItem() //列表页采集结果
|
|
|
+ if downloadNum == 0 {
|
|
|
+ f.ServeJson(steps)
|
|
|
+ return
|
|
|
}
|
|
|
- if len(list) > 0 {
|
|
|
- tr.listInfo = list
|
|
|
- listone := list[0]
|
|
|
- if len(qu.ObjToString(listone["href"])) < 7 ||
|
|
|
- (qu.ObjToString(listone["publishtime"]) != "0" && len(qu.ObjToString(listone["publishtime"])) < 5) ||
|
|
|
- len(qu.ObjToString(listone["title"])) < 3 {
|
|
|
- f.ServeJson(steps)
|
|
|
- return
|
|
|
- } else {
|
|
|
- steps[2] = true
|
|
|
- if s.DownDetail {
|
|
|
- param := map[string]string{}
|
|
|
- index := 0
|
|
|
- if len(list) > 0 {
|
|
|
- steps[8] = len(list)
|
|
|
- index = len(list) / 2
|
|
|
- for k, v := range list[index] {
|
|
|
- param[k] = qu.ObjToString(v)
|
|
|
- }
|
|
|
- data = map[string]interface{}{}
|
|
|
- s.DownloadDetailPage(param, data)
|
|
|
- if len(data) > 0 {
|
|
|
- tr.dataInfo = data
|
|
|
- }
|
|
|
- if len(data) == 0 || data["detail"].(string) == "" {
|
|
|
- steps[3] = false
|
|
|
- } else {
|
|
|
- steps[3] = true
|
|
|
- }
|
|
|
- }
|
|
|
+ steps[2] = true //列表页获取成功
|
|
|
+ steps[8] = downloadNum //下载量
|
|
|
+ tr.listInfo = result
|
|
|
+ if s.DownDetail {
|
|
|
+ onePageList := result[1] //第一页数据
|
|
|
+ if onePageDataNum := len(onePageList); onePageDataNum > 0 {
|
|
|
+ index := onePageDataNum / 2 //取一条数据下载三级页
|
|
|
+ param := map[string]string{}
|
|
|
+ for k, v := range onePageList[index] {
|
|
|
+ param[k] = qu.ObjToString(v)
|
|
|
+ }
|
|
|
+ data = map[string]interface{}{}
|
|
|
+ s.DownloadDetailPage(param, data)
|
|
|
+ tr.dataInfo = data
|
|
|
+ if len(data) == 0 || qu.ObjToString(data["detail"]) == "" {
|
|
|
+ steps[3] = false //详情页获取失败
|
|
|
} else {
|
|
|
- steps[3] = true
|
|
|
+ steps[3] = true //详情页获取成功
|
|
|
}
|
|
|
}
|
|
|
+ } else {
|
|
|
+ steps[3] = true //详情页获取成功
|
|
|
}
|
|
|
+ //list, _ = s.DownListPageItem()
|
|
|
+ //for _, l := range list {
|
|
|
+ // if publishtime := qu.ObjToString(l["publishtime"]); publishtime == "0" || publishtime == "" {
|
|
|
+ // msgArr = append(msgArr, "列表页publishtime取值异常")
|
|
|
+ // break
|
|
|
+ // } else {
|
|
|
+ // t, err := time.ParseInLocation(qu.Date_Full_Layout, publishtime, time.Local)
|
|
|
+ // if err != nil || t.Unix() <= 0 {
|
|
|
+ // msgArr = append(msgArr, "列表页publishtime取值异常")
|
|
|
+ // break
|
|
|
+ // }
|
|
|
+ // }
|
|
|
+ //}
|
|
|
+ //if len(list) > 0 {
|
|
|
+ // tr.listInfo = list
|
|
|
+ // listone := list[0]
|
|
|
+ // if len(qu.ObjToString(listone["href"])) < 7 ||
|
|
|
+ // (qu.ObjToString(listone["publishtime"]) != "0" && len(qu.ObjToString(listone["publishtime"])) < 5) ||
|
|
|
+ // len(qu.ObjToString(listone["title"])) < 3 {
|
|
|
+ // f.ServeJson(steps)
|
|
|
+ // return
|
|
|
+ // } else {
|
|
|
+ // steps[2] = true
|
|
|
+ // if s.DownDetail {
|
|
|
+ // param := map[string]string{}
|
|
|
+ // index := 0
|
|
|
+ // if len(list) > 0 {
|
|
|
+ // steps[8] = len(list)
|
|
|
+ // index = len(list) / 2
|
|
|
+ // for k, v := range list[index] {
|
|
|
+ // param[k] = qu.ObjToString(v)
|
|
|
+ // }
|
|
|
+ // data = map[string]interface{}{}
|
|
|
+ // s.DownloadDetailPage(param, data)
|
|
|
+ // if len(data) > 0 {
|
|
|
+ // tr.dataInfo = data
|
|
|
+ // }
|
|
|
+ // if len(data) == 0 || data["detail"].(string) == "" {
|
|
|
+ // steps[3] = false
|
|
|
+ // } else {
|
|
|
+ // steps[3] = true
|
|
|
+ // }
|
|
|
+ // }
|
|
|
+ // } else {
|
|
|
+ // steps[3] = true
|
|
|
+ // }
|
|
|
+ // }
|
|
|
+ //}
|
|
|
}
|
|
|
//关闭laustate
|
|
|
s.L.Close()
|
|
@@ -852,7 +897,7 @@ func (f *Front) SpiderPass() {
|
|
|
steps[6] = s.Test_goreqlist
|
|
|
steps[7] = s.Test_goreqcon
|
|
|
//校验
|
|
|
- msg := u.SpiderPassCheckListAndDetail(list, data)
|
|
|
+ msg := u.SpiderPassCheckListAndDetail(result, data)
|
|
|
msgArr = append(msgArr, msg...)
|
|
|
username := f.GetSession("username").(string)
|
|
|
tr.msg = strings.Join(msgArr, ";")
|