|
|
@@ -13,7 +13,7 @@ import (
|
|
|
|
|
|
mu "mfw/util"
|
|
|
qu "qfw/util"
|
|
|
- util "spiderutil"
|
|
|
+ sp "spiderutil"
|
|
|
"strings"
|
|
|
"time"
|
|
|
u "util"
|
|
|
@@ -56,6 +56,7 @@ type Step2 struct {
|
|
|
DateFormat string
|
|
|
Expert string
|
|
|
Types int
|
|
|
+ Chrome string
|
|
|
}
|
|
|
|
|
|
type Step3 struct {
|
|
|
@@ -66,6 +67,7 @@ type Step3 struct {
|
|
|
T_date string
|
|
|
Expert string
|
|
|
Types int
|
|
|
+ Chrome string
|
|
|
}
|
|
|
|
|
|
type StepRe3 struct {
|
|
|
@@ -101,7 +103,7 @@ func (f *Front) LoadSpider(codeTaskIdReState string) error {
|
|
|
}
|
|
|
copy := f.GetString("copy")
|
|
|
if f.Method() == "GET" {
|
|
|
- code := util.Se.Decode4Hex(code)
|
|
|
+ code := sp.Se.Decode4Hex(code)
|
|
|
f.T["actiontext"] = "编辑"
|
|
|
//lua, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": code})
|
|
|
lua, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
|
|
|
@@ -169,7 +171,7 @@ func (f *Front) LoadSpider(codeTaskIdReState string) error {
|
|
|
f.T["spidermovevent"] = (*lua)["spidermovevent"]
|
|
|
f.T["spiderhistorymaxpage"] = (*lua)["spiderhistorymaxpage"]
|
|
|
events := []string{}
|
|
|
- for k, _ := range util.Config.Uploadevents {
|
|
|
+ for k, _ := range sp.Config.Uploadevents {
|
|
|
events = append(events, k)
|
|
|
}
|
|
|
sort.Strings(events)
|
|
|
@@ -191,7 +193,7 @@ func (f *Front) ViewSpider(id string) error {
|
|
|
auth := qu.IntAll(f.GetSession("auth"))
|
|
|
if auth >= 1 {
|
|
|
if f.Method() == "GET" {
|
|
|
- code := util.Se.Decode4Hex(id)
|
|
|
+ code := sp.Se.Decode4Hex(id)
|
|
|
f.T["actiontext"] = "编辑"
|
|
|
//lua, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": code})
|
|
|
lua, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
|
|
|
@@ -216,7 +218,7 @@ func (f *Front) ViewSpider(id string) error {
|
|
|
f.T["spidermovevent"] = (*lua)["spidermovevent"]
|
|
|
f.T["spiderhistorymaxpage"] = (*lua)["spiderhistorymaxpage"]
|
|
|
events := []string{}
|
|
|
- for k, _ := range util.Config.Uploadevents {
|
|
|
+ for k, _ := range sp.Config.Uploadevents {
|
|
|
events = append(events, k)
|
|
|
}
|
|
|
sort.Strings(events)
|
|
|
@@ -249,9 +251,14 @@ func (f *Front) LoadModel(id string) error {
|
|
|
}
|
|
|
|
|
|
func (f *Front) SaveStep() {
|
|
|
+ rep := map[string]interface{}{}
|
|
|
+ if f.Step2.Types == 2 || f.Step3.Types == 2 { //chrome模式只支持转成专家模式保存
|
|
|
+ rep["msg"] = "爬虫不支持chrome模式保存"
|
|
|
+ f.ServeJson(rep)
|
|
|
+ return
|
|
|
+ }
|
|
|
userid, _ := f.GetSession("userid").(string)
|
|
|
auth := qu.IntAll(f.GetSession("auth"))
|
|
|
- rep := map[string]interface{}{}
|
|
|
if f.GetString("oldlua") != "" {
|
|
|
id := f.GetString("code")
|
|
|
//one, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": id})
|
|
|
@@ -269,7 +276,7 @@ func (f *Front) SaveStep() {
|
|
|
b := u.MgoEB.Update("luaconfig", map[string]interface{}{"code": id}, map[string]interface{}{"$set": upset}, true, false)
|
|
|
if b {
|
|
|
rep["msg"] = "保存成功"
|
|
|
- rep["code"] = util.Se.Encode2Hex(id)
|
|
|
+ rep["code"] = sp.Se.Encode2Hex(id)
|
|
|
f.ServeJson(rep)
|
|
|
return
|
|
|
}
|
|
|
@@ -343,6 +350,9 @@ func (f *Front) SaveStep() {
|
|
|
f.Step3.ContentChooser,
|
|
|
f.Step3.ElementChooser,
|
|
|
}
|
|
|
+ param_list_chrome, param_content_chrome := []sp.ChromeActions{}, []sp.ChromeActions{}
|
|
|
+ json.Unmarshal([]byte(f.Step2.Chrome), ¶m_list_chrome)
|
|
|
+ json.Unmarshal([]byte(f.Step3.Chrome), ¶m_content_chrome)
|
|
|
param := map[string]interface{}{}
|
|
|
common[4] = 1
|
|
|
param["param_common"] = common
|
|
|
@@ -351,13 +361,18 @@ func (f *Front) SaveStep() {
|
|
|
//向导模式
|
|
|
param["param_time"] = ptime
|
|
|
param["param_list"] = list
|
|
|
+ param["param_list_chrome"] = param_list_chrome
|
|
|
+ param["param_content_chrome"] = param_content_chrome
|
|
|
param["param_content"] = content
|
|
|
param["type_time"] = f.Step1.Types
|
|
|
param["type_list"] = f.Step2.Types
|
|
|
param["type_content"] = f.Step3.Types
|
|
|
+
|
|
|
//专家模式
|
|
|
param["str_time"] = f.Step1.Expert
|
|
|
param["str_list"] = f.Step2.Expert
|
|
|
+ param["str_list_chrome"] = f.Step2.Chrome
|
|
|
+ param["str_content_chrome"] = f.Step3.Chrome
|
|
|
param["str_content"] = f.Step3.Expert
|
|
|
param["comeintime"] = comeintime
|
|
|
listcheck = strings.Replace(listcheck, "\n", "\\\\n", -1)
|
|
|
@@ -366,7 +381,7 @@ func (f *Front) SaveStep() {
|
|
|
param["contentcheck"] = strings.Replace(contentcheck, "\"", "\\\\\"", -1)
|
|
|
//补充模型
|
|
|
s_model := f.GetString("model")
|
|
|
- configModel := util.Config.Model[s_model]
|
|
|
+ configModel := sp.Config.Model[s_model]
|
|
|
model := map[string]interface{}{}
|
|
|
for k, _ := range configModel {
|
|
|
model[k] = f.GetString(k)
|
|
|
@@ -421,7 +436,7 @@ func (f *Front) SaveStep() {
|
|
|
} else if f.OtherBase.SpiderType == "increment" && err == nil { //增量
|
|
|
param["event"] = tmpEvent //开发人员切换增量节点
|
|
|
}
|
|
|
- if movevent, ok := util.Config.Uploadevents[f.OtherBase.SpiderMoveEvent].(string); ok && movevent != "" {
|
|
|
+ if movevent, ok := sp.Config.Uploadevents[f.OtherBase.SpiderMoveEvent].(string); ok && movevent != "" {
|
|
|
param["spidermovevent"] = movevent
|
|
|
}
|
|
|
//开发人员修改爬虫节点后，在审核人员上架时，要在原来的节点下架，临时记录要下架的节点downevent
|
|
|
@@ -433,7 +448,8 @@ func (f *Front) SaveStep() {
|
|
|
param["iscopycontent"] = f.StepRe3.Checked
|
|
|
//
|
|
|
param["listisfilter"] = ListFilterReg.MatchString(f.Step2.Expert) //列表页校验是否含“--关键词过滤”
|
|
|
- checkLua := LuaTextCheck(f.Base.SpiderCode, f.Step2.Expert, f.Step3.Expert, f.Step2.Types, infoformat, model, msgResult)
|
|
|
+ checkLua := LuaTextCheck(infoformat, param, param_list_chrome, param_content_chrome, msgResult)
|
|
|
+ //checkLua := LuaTextCheck(f.Base.SpiderCode, f.Step2.Expert, f.Step3.Expert, f.Step2.Types, infoformat, model, msgResult)
|
|
|
if !checkLua {
|
|
|
issave := spider.SaveSpider(code, param) //保存脚本
|
|
|
if issave {
|
|
|
@@ -455,7 +471,7 @@ func (f *Front) SaveStep() {
|
|
|
rep["msg"] = "保存失败，" + msgResult["err"]
|
|
|
rep["ok"] = false
|
|
|
}
|
|
|
- rep["code"] = util.Se.Encode2Hex(code)
|
|
|
+ rep["code"] = sp.Se.Encode2Hex(code)
|
|
|
f.ServeJson(rep)
|
|
|
}
|
|
|
}
|
|
|
@@ -549,7 +565,7 @@ func LuaSaveLog(code, user string, data *map[string]interface{}, stype int) {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-//爬虫保存时，检查列表页和三级页代码中是否含lua原生方法
|
|
|
+/*爬虫保存时，检查列表页和三级页代码中是否含lua原生方法
|
|
|
func LuaTextCheck(code, list, detail string, type_list, infoformat int, model map[string]interface{}, msgResult map[string]string) bool {
|
|
|
defer qu.Catch()
|
|
|
//1、异常校验
|
|
|
@@ -616,6 +632,94 @@ func LuaTextCheck(code, list, detail string, type_list, infoformat int, model ma
|
|
|
msgResult["warn"] += warnmsg
|
|
|
msgResult["err"] = errmsg
|
|
|
return errmsg != ""
|
|
|
+}*/
|
|
|
+func LuaTextCheck(infoformat int, param map[string]interface{}, param_list_chrome, param_content_chrome []sp.ChromeActions, msgResult map[string]string) bool {
|
|
|
+ code := qu.ObjToString(param["code"])
|
|
|
+ list := qu.ObjToString(param["str_list"])
|
|
|
+ detail := qu.ObjToString(param["str_content"])
|
|
|
+ type_list := qu.IntAll(param["type_list"])
|
|
|
+ type_content := qu.IntAll(param["type_content"])
|
|
|
+ model, _ := param["model"].(map[string]interface{})
|
|
|
+ defer qu.Catch()
|
|
|
+ //1、异常校验
|
|
|
+ var errmsg, warnmsg string
|
|
|
+ if LuaReg.MatchString(list) || LuaReg.MatchString(detail) {
|
|
|
+ errmsg += "代码中含有lua原生方法；"
|
|
|
+ }
|
|
|
+ if ListFilterReg.MatchString(detail) && !strings.Contains(detail, "delete") { //三级页含过滤但是没有data["delete"]="true"
|
|
|
+ errmsg += `三级页缺少data["delete"]="true"；`
|
|
|
+ }
|
|
|
+ sln_reg := regexp.MustCompile(`sendListNum\(pageno,list\)`)
|
|
|
+ slnIndexArr := sln_reg.FindAllStringIndex(list, -1)
|
|
|
+ if type_list == 1 {
|
|
|
+ if strings.Contains(list, "downloadByChrome") { //chrome下载方法动作参数判断
|
|
|
+ for _, act := range param_list_chrome {
|
|
|
+ if act.Action != "changeip" && act.Param == "" {
|
|
|
+ errmsg += "列表页chrome模式'" + act.Action + "'动作未填写参数，填写后注意Ctrl+F10重新插入代码；"
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if len(slnIndexArr) == 0 { //列表页专家模式且不含sendListNum
|
|
|
+ errmsg += "代码中缺少sendListNum(pageno,list)方法；"
|
|
|
+ } else if len(slnIndexArr) > 0 { //判断sendListNum方法的位置
|
|
|
+ trim_reg := regexp.MustCompile("trim")
|
|
|
+ insert_reg := regexp.MustCompile("insert")
|
|
|
+ trIndexArr := trim_reg.FindAllStringIndex(list, -1)
|
|
|
+ irIndexArr := insert_reg.FindAllStringIndex(list, -1)
|
|
|
+ slIndex := slnIndexArr[len(slnIndexArr)-1] //sendListNum位置
|
|
|
+ trIndex := trIndexArr[len(trIndexArr)-1] //com.trim位置
|
|
|
+ irIndex := irIndexArr[len(irIndexArr)-1] //insert位置
|
|
|
+ qu.Debug("sendListNum位置：", trIndex, slIndex, irIndex)
|
|
|
+ if slIndex[1] < trIndex[0] || slIndex[0] > irIndex[1] { //sendListNum方法必须在com.trim方法后，table.insert方法前
|
|
|
+ errmsg += "sendListNum方法位置错误；"
|
|
|
+ }
|
|
|
+ }
|
|
|
+ //校验列表页area、city、distric
|
|
|
+ if !strings.Contains(list, "area") {
|
|
|
+ errmsg += `模板item["area"]不存在；`
|
|
|
+ }
|
|
|
+ if !strings.Contains(list, "city") {
|
|
|
+ errmsg += `模板item["city"]不存在；`
|
|
|
+ }
|
|
|
+ if !strings.Contains(list, "district") {
|
|
|
+ errmsg += `模板item["district"]不存在；`
|
|
|
+ }
|
|
|
+ area := qu.ObjToString(model["area"])
|
|
|
+ city := qu.ObjToString(model["city"])
|
|
|
+ district := qu.ObjToString(model["district"])
|
|
|
+ if area != "" && !strings.Contains(list, area) {
|
|
|
+ errmsg += "省份信息与模板不一致；"
|
|
|
+ }
|
|
|
+ if city != "" && !strings.Contains(list, city) {
|
|
|
+ errmsg += "城市信息与模板不一致；"
|
|
|
+ }
|
|
|
+ if district != "" && !strings.Contains(list, district) {
|
|
|
+ errmsg += "区/县信息与模板不一致；"
|
|
|
+ }
|
|
|
+ if infoformat == 2 && !strings.Contains(detail, "projectname") {
|
|
|
+ errmsg += "拟建/审批数据缺少projectname字段；"
|
|
|
+ }
|
|
|
+ //校验爬虫代码的一致性
|
|
|
+ if !strings.Contains(list, code) {
|
|
|
+ errmsg += `模板item["spidercode"]值错误；`
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if type_content == 1 {
|
|
|
+ if strings.Contains(detail, "downloadByChrome") { //chrome下载方法动作参数判断
|
|
|
+ for _, act := range param_content_chrome {
|
|
|
+ if act.Action != "changeip" && act.Param == "" {
|
|
|
+ errmsg += "详情页chrome模式'" + act.Action + "'动作未填写参数，填写后注意Ctrl+F10重新插入代码；"
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ //2、提醒校验
|
|
|
+ if !strings.Contains(detail, "downloadFile") && !strings.Contains(detail, "getFileAttachmentsArrayWithTag") {
|
|
|
+ warnmsg += "三级页缺少下载附件方法；"
|
|
|
+ }
|
|
|
+ msgResult["warn"] += warnmsg
|
|
|
+ msgResult["err"] = errmsg
|
|
|
+ return errmsg != ""
|
|
|
}
|
|
|
|
|
|
//方法测试
|
|
|
@@ -647,7 +751,7 @@ func (f *Front) RunStep() {
|
|
|
}
|
|
|
if f.Method() == "POST" {
|
|
|
switch f.GetString("step") {
|
|
|
- case "step1": //publishtime
|
|
|
+ case "Step1": //publishtime
|
|
|
ptime := []interface{}{
|
|
|
f.Step1.DateFormat,
|
|
|
f.Step1.Address,
|
|
|
@@ -662,78 +766,108 @@ func (f *Front) RunStep() {
|
|
|
if err == nil {
|
|
|
f.ServeJson(rs)
|
|
|
}
|
|
|
- case "step2": //list
|
|
|
- addrs := strings.Split(f.Step2.Listadds, "\n")
|
|
|
- if len(addrs) > 0 {
|
|
|
- for k, v := range addrs {
|
|
|
- addrs[k] = "'" + v + "'"
|
|
|
+ case "Step2": //list
|
|
|
+ if imodal == 2 { //chromedp测试
|
|
|
+ chromeActions := []sp.ChromeActions{}
|
|
|
+ var result []interface{}
|
|
|
+ if json.Unmarshal([]byte(f.Step2.Chrome), &chromeActions) == nil {
|
|
|
+ chromeTask := sp.ChromeTask{
|
|
|
+ TimeOut: 120, //测试默认两分钟
|
|
|
+ Actions: chromeActions,
|
|
|
+ }
|
|
|
+ result = spider.DownloadByChrome(downloadnode, "", chromeTask, 150)
|
|
|
+ } else {
|
|
|
+ result = append(result, "chrome task json 格式化错误")
|
|
|
}
|
|
|
- f.Step2.Listadds = strings.Join(addrs, ",")
|
|
|
- } else if len(f.Step2.Listadds) > 5 {
|
|
|
- f.Step2.Listadds = "'" + f.Step2.Listadds + "'"
|
|
|
+ f.ServeJson(result)
|
|
|
} else {
|
|
|
- f.Step2.Listadds = ""
|
|
|
- }
|
|
|
- list := []interface{}{
|
|
|
- f.Step2.Listadd,
|
|
|
- f.Step2.Listadds,
|
|
|
- f.Step2.BlockChooser,
|
|
|
- f.Step2.AddressChooser,
|
|
|
- f.Step2.TitleChooser,
|
|
|
- f.Step2.DateChooser,
|
|
|
- f.Step2.DateFormat,
|
|
|
- }
|
|
|
- listcheck = strings.Replace(listcheck, "\n", "\\n", -1)
|
|
|
- listcheck = strings.Replace(listcheck, "\"", "\\\"", -1)
|
|
|
- s_model := f.GetString("model")
|
|
|
- configModel := util.Config.Model[s_model]
|
|
|
- model := map[string]interface{}{}
|
|
|
- for k, _ := range configModel {
|
|
|
- model[k] = f.GetString(k)
|
|
|
- }
|
|
|
- if script {
|
|
|
- _, script := spider.GetPageList(common, list, model, listcheck, f.Step2.Expert, downloadnode, imodal, 1)
|
|
|
- f.ServeJson(script)
|
|
|
- return
|
|
|
- }
|
|
|
- rs, err := spider.GetPageList(common, list, model, listcheck, f.Step2.Expert, downloadnode, imodal)
|
|
|
- if err == nil {
|
|
|
- f.ServeJson(rs)
|
|
|
- } else if err.(error).Error() == "no" {
|
|
|
- f.ServeJson(rs[0])
|
|
|
- }
|
|
|
- case "step3": //detail
|
|
|
- content := []interface{}{
|
|
|
- f.Step3.ContentChooser,
|
|
|
- f.Step3.ElementChooser,
|
|
|
+ addrs := strings.Split(f.Step2.Listadds, "\n")
|
|
|
+ if len(addrs) > 0 {
|
|
|
+ for k, v := range addrs {
|
|
|
+ addrs[k] = "'" + v + "'"
|
|
|
+ }
|
|
|
+ f.Step2.Listadds = strings.Join(addrs, ",")
|
|
|
+ } else if len(f.Step2.Listadds) > 5 {
|
|
|
+ f.Step2.Listadds = "'" + f.Step2.Listadds + "'"
|
|
|
+ } else {
|
|
|
+ f.Step2.Listadds = ""
|
|
|
+ }
|
|
|
+ list := []interface{}{
|
|
|
+ f.Step2.Listadd,
|
|
|
+ f.Step2.Listadds,
|
|
|
+ f.Step2.BlockChooser,
|
|
|
+ f.Step2.AddressChooser,
|
|
|
+ f.Step2.TitleChooser,
|
|
|
+ f.Step2.DateChooser,
|
|
|
+ f.Step2.DateFormat,
|
|
|
+ }
|
|
|
+ listcheck = strings.Replace(listcheck, "\n", "\\n", -1)
|
|
|
+ listcheck = strings.Replace(listcheck, "\"", "\\\"", -1)
|
|
|
+ s_model := f.GetString("model")
|
|
|
+ configModel := sp.Config.Model[s_model]
|
|
|
+ model := map[string]interface{}{}
|
|
|
+ for k, _ := range configModel {
|
|
|
+ model[k] = f.GetString(k)
|
|
|
+ }
|
|
|
+ if script {
|
|
|
+ _, script := spider.GetPageList(common, list, model, listcheck, f.Step2.Expert, downloadnode, imodal, 1)
|
|
|
+ f.ServeJson(script)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ rs, err := spider.GetPageList(common, list, model, listcheck, f.Step2.Expert, downloadnode, imodal)
|
|
|
+ if err == nil {
|
|
|
+ f.ServeJson(rs)
|
|
|
+ } else if err.(error).Error() == "no" {
|
|
|
+ f.ServeJson(rs[0])
|
|
|
+ }
|
|
|
}
|
|
|
+ case "Step3": //detail
|
|
|
+ if imodal == 2 { //chromedp测试
|
|
|
+ chromeActions := []sp.ChromeActions{}
|
|
|
+ var result []interface{}
|
|
|
+ if json.Unmarshal([]byte(f.Step3.Chrome), &chromeActions) == nil {
|
|
|
+ chromeTask := sp.ChromeTask{
|
|
|
+ TimeOut: 120, //测试默认两分钟
|
|
|
+ Actions: chromeActions,
|
|
|
+ }
|
|
|
+ result = spider.DownloadByChrome(downloadnode, "", chromeTask, 150)
|
|
|
+ } else {
|
|
|
+ result = append(result, "chrome task json 格式化错误")
|
|
|
+ }
|
|
|
+ f.ServeJson(result)
|
|
|
+ } else {
|
|
|
+ content := []interface{}{
|
|
|
+ f.Step3.ContentChooser,
|
|
|
+ f.Step3.ElementChooser,
|
|
|
+ }
|
|
|
|
|
|
- contentcheck = strings.Replace(contentcheck, "\n", "\\n", -1)
|
|
|
- contentcheck = strings.Replace(contentcheck, "\"", "\\\"", -1)
|
|
|
- data := map[string]interface{}{}
|
|
|
- data["title"] = f.Step3.T_title
|
|
|
- data["href"] = f.Step3.T_href
|
|
|
- data["publishtime"] = f.Step3.T_date
|
|
|
- if script {
|
|
|
- _, script := spider.GetContentInfo(common, content, data, contentcheck, f.Step3.Expert, downloadnode, imodal, 1)
|
|
|
- f.ServeJson(script)
|
|
|
- return
|
|
|
- }
|
|
|
- rs, err := spider.GetContentInfo(common, content, data, contentcheck, f.Step3.Expert, downloadnode, imodal)
|
|
|
- if projectinfo, ok := rs["projectinfo"].(map[string]interface{}); ok && projectinfo != nil {
|
|
|
- if attachments, ok := projectinfo["attachments"].(map[string]interface{}); ok && attachments != nil {
|
|
|
- for _, tmp := range attachments {
|
|
|
- tmpMap := tmp.(map[string]interface{})
|
|
|
- if qu.ObjToString(tmpMap["filename"]) == "附件中含有乱码" {
|
|
|
- rs["msg"] = "附件中含有乱码"
|
|
|
+ contentcheck = strings.Replace(contentcheck, "\n", "\\n", -1)
|
|
|
+ contentcheck = strings.Replace(contentcheck, "\"", "\\\"", -1)
|
|
|
+ data := map[string]interface{}{}
|
|
|
+ data["title"] = f.Step3.T_title
|
|
|
+ data["href"] = f.Step3.T_href
|
|
|
+ data["publishtime"] = f.Step3.T_date
|
|
|
+ if script {
|
|
|
+ _, script := spider.GetContentInfo(common, content, data, contentcheck, f.Step3.Expert, downloadnode, imodal, 1)
|
|
|
+ f.ServeJson(script)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ rs, err := spider.GetContentInfo(common, content, data, contentcheck, f.Step3.Expert, downloadnode, imodal)
|
|
|
+ if projectinfo, ok := rs["projectinfo"].(map[string]interface{}); ok && projectinfo != nil {
|
|
|
+ if attachments, ok := projectinfo["attachments"].(map[string]interface{}); ok && attachments != nil {
|
|
|
+ for _, tmp := range attachments {
|
|
|
+ tmpMap := tmp.(map[string]interface{})
|
|
|
+ if qu.ObjToString(tmpMap["filename"]) == "附件中含有乱码" {
|
|
|
+ rs["msg"] = "附件中含有乱码"
|
|
|
+ }
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
- }
|
|
|
- if err == nil {
|
|
|
- f.ServeJson(rs)
|
|
|
- } else {
|
|
|
- f.ServeJson(rs["no"])
|
|
|
+ if err == nil {
|
|
|
+ f.ServeJson(rs)
|
|
|
+ } else {
|
|
|
+ f.ServeJson(rs["no"])
|
|
|
+ }
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
@@ -1122,7 +1256,7 @@ func (f *Front) UpState() error {
|
|
|
res := map[string]interface{}{
|
|
|
"istotask": istotask,
|
|
|
"err": "没有权限",
|
|
|
- "code": util.Se.Encode2Hex(code),
|
|
|
+ "code": sp.Se.Encode2Hex(code),
|
|
|
"taskid": taskid,
|
|
|
}
|
|
|
var xgTime int64
|
|
|
@@ -1247,12 +1381,12 @@ func UpStateAndUpSpider(code, id, reason, username string, state int) (bool, err
|
|
|
if (*one)["event"] != nil {
|
|
|
event = qu.IntAll((*one)["event"])
|
|
|
} else {
|
|
|
- for k, _ := range util.Config.Uploadevents { //
|
|
|
+ for k, _ := range sp.Config.Uploadevents { //
|
|
|
event = qu.IntAll(k)
|
|
|
break
|
|
|
}
|
|
|
//r := rand.New(rand.NewSource(time.Now().UnixNano()))
|
|
|
- //event = util.Config.Uploadevents[r.Intn(len(util.Config.Uploadevents))]
|
|
|
+ //event = sp.Config.Uploadevents[r.Intn(len(sp.Config.Uploadevents))]
|
|
|
}
|
|
|
//oldstate := qu.IntAll(one["state"])
|
|
|
switch state {
|
|
|
@@ -1481,7 +1615,7 @@ func (f *Front) ChangeEvent() {
|
|
|
code := f.GetString("code")
|
|
|
event, _ := f.GetInt("event")
|
|
|
eventok := false
|
|
|
- for k, _ := range util.Config.Uploadevents {
|
|
|
+ for k, _ := range sp.Config.Uploadevents {
|
|
|
if event == qu.Int64All(k) {
|
|
|
eventok = true
|
|
|
break
|
|
|
@@ -1643,7 +1777,7 @@ func (f *Front) LuaList() {
|
|
|
f.ServeJson(map[string]interface{}{"draw": draw, "data": luas, "recordsFiltered": count, "recordsTotal": count})
|
|
|
} else {
|
|
|
events := []string{}
|
|
|
- for k, _ := range util.Config.Uploadevents {
|
|
|
+ for k, _ := range sp.Config.Uploadevents {
|
|
|
events = append(events, k)
|
|
|
}
|
|
|
sort.Strings(events)
|
|
|
@@ -1728,7 +1862,7 @@ func (f *Front) Heart() {
|
|
|
f.ServeJson(map[string]interface{}{"draw": draw, "data": list, "recordsFiltered": count, "recordsTotal": count})
|
|
|
} else {
|
|
|
events := []string{}
|
|
|
- for k, _ := range util.Config.Uploadevents {
|
|
|
+ for k, _ := range sp.Config.Uploadevents {
|
|
|
events = append(events, k)
|
|
|
}
|
|
|
sort.Strings(events)
|
|
|
@@ -1751,7 +1885,7 @@ type spinfo struct {
|
|
|
|
|
|
//爬虫信息
|
|
|
func SpiderInfo(data string) {
|
|
|
- data = util.Se.DecodeString(data)
|
|
|
+ data = sp.Se.DecodeString(data)
|
|
|
infos := []map[string]interface{}{}
|
|
|
err := json.Unmarshal([]byte(data), &infos)
|
|
|
if err != nil {
|
|
|
@@ -1781,7 +1915,7 @@ func SpiderInfo(data string) {
|
|
|
|
|
|
//接受维护任务信息
|
|
|
func SpiderModifyTask(data string) {
|
|
|
- data = util.Se.DecodeString(data)
|
|
|
+ data = sp.Se.DecodeString(data)
|
|
|
mtasks := []map[string]interface{}{}
|
|
|
err := json.Unmarshal([]byte(data), &mtasks)
|
|
|
if err != nil {
|
