功能调整

src/commutil/codesync.go

@@ -16,6 +16,32 @@ import (
 	"strings"
 )
 
+// 任务发布
+func publishModule(taskCode, scriptCode, params string, workStart, workEnd, workTimeout, interval int) (err error) {
+	if workStart < 0 || workEnd < 0 || workStart > workEnd || workEnd > 23 {
+		return errors.New("起始或结束时间异常！")
+	} else {
+		req, err := http.NewRequest("GET", fmt.Sprintf("%s/publish", sp.Config.TaskMgrSerever), nil)
+		if err != nil {
+			return errors.New("服务端链接创建失败！")
+		}
+		req.Header.Set("task_code", taskCode)
+		req.Header.Set("script_code", scriptCode)
+		req.Header.Set("work_start", strconv.Itoa(workStart))
+		req.Header.Set("work_end", strconv.Itoa(workEnd))
+		req.Header.Set("worktimeout", strconv.Itoa(workTimeout))
+		req.Header.Set("interval", strconv.Itoa(interval))
+		req.Header.Set("params", params)
+		client := httpClient()
+		resp, err := client.Do(req)
+		if err != nil {
+			return errors.New("服务端发布请求失败！")
+		}
+		defer resp.Body.Close()
+	}
+	return
+}
+
 // postModule 推送模块
 func postModule(path, module string) error {
 	name := module
@@ -72,32 +98,6 @@ func postModule(path, module string) error {
 	return err
 }
 
-// 任务发布
-func publishModule(taskCode, scriptCode, params string, workStart, workEnd, workTimeout, interval int) (err error) {
-	if workStart < 0 || workEnd < 0 || workStart > workEnd || workEnd > 23 {
-		return errors.New("起始或结束时间异常！")
-	} else {
-		req, err := http.NewRequest("GET", fmt.Sprintf("%s/publish", sp.Config.TaskMgrSerever), nil)
-		if err != nil {
-			return errors.New("服务端链接创建失败！")
-		}
-		req.Header.Set("task_code", taskCode)
-		req.Header.Set("script_code", scriptCode)
-		req.Header.Set("work_start", strconv.Itoa(workStart))
-		req.Header.Set("work_end", strconv.Itoa(workEnd))
-		req.Header.Set("worktimeout", strconv.Itoa(workTimeout))
-		req.Header.Set("interval", strconv.Itoa(interval))
-		req.Header.Set("params", params)
-		client := httpClient()
-		resp, err := client.Do(req)
-		if err != nil {
-			return errors.New("服务端发布请求失败！")
-		}
-		defer resp.Body.Close()
-	}
-	return
-}
-
 // unPubModule 撤销任务发布
 func unPubModule(taskCode string) error {
 	req, err := http.NewRequest("GET", fmt.Sprintf("%s/unpublish", sp.Config.CodeDbSerever), nil)

src/commutil/commcodecheck.go

@@ -1,18 +1,5 @@
 package commutil
 
-import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	qu "qfw/util"
-	sp "spiderutil"
-	"strings"
-	"sync"
-	"time"
-	"util"
-)
-
 var (
 	CommCodesCache = make(chan map[string]interface{}, 5000)
 )
@@ -68,7 +55,7 @@ type (
 	}
 )
 
-func CommCodeCheck() {
+/*func CommCodeCheck() {
 	wg := &sync.WaitGroup{}
 	ch := make(chan bool, 5)
 	for {
@@ -80,71 +67,21 @@ func CommCodeCheck() {
 				<-ch
 				wg.Done()
 			}()
-			href := qu.ObjToString(tmp["href"])
-			stype := qu.ObjToString(tmp["stype"])
-			r, err := OmnipotentCodeCheck(href, stype)
-			omnipotent, _ := r["omnipotent"].(bool)
-			if err == nil {
-				r["code"] = tmp["code"]
-				r["site"] = tmp["site"]
-				r["channel"] = tmp["channel"]
-				r["comeintime"] = time.Now().Unix()
-				r["href"] = href
-				util.MgoEB.Save("luaconfig_omnipotent_log", r)
-				set := map[string]interface{}{}
-				if omnipotent {
-					set["cssmark"] = r["cssmark"]
-					set["omnipotent"] = true
-					set["recovertime"] = time.Now().Unix()
-					set["claimtime"] = time.Now().Unix()
-					set["claimtype"] = 2 //已认领
-					set["state"] = 11    //已上线
-					//默认机器人用户
-					set["createuser"] = UserName
-					set["createuserid"] = UserId
-					set["createuseremail"] = UserEmail
-					set["modifyuser"] = UserName
-					set["modifyuserid"] = UserId
-				} else if qu.IntAll(r["flag"]) == PlatformFlag {
-					set["platform"] = "golua平台"
-					set["incrementevent"] = 7200
-				}
-				if len(set) > 0 {
-					util.MgoEB.Update("luaconfig",
-						map[string]interface{}{
-							"code":      tmp["code"],
-							"claimtype": 0,
-						},
-						map[string]interface{}{
-							"$set": set,
-						},
-						false, false)
-					if omnipotent {
-						time.Sleep(5 * time.Second)
-						one, _ := util.MgoEB.FindOneByField("luaconfig",
-							map[string]interface{}{"code": tmp["code"]},
-							map[string]interface{}{"str_list": 0, "str_content": 0},
-						)
-						if len(*one) > 0 { //保存线上表
-							util.MgoEB.SaveByOriID("luaconfig_online", *one)
-							err = CommCodeMgr(*one)
-							if err != nil {
-								qu.Debug("爬虫发布失败：", err.Error())
-							}
-						} else {
-							qu.Debug("爬虫查询异常，发布失败：", tmp["code"])
-						}
-					}
-				}
-			} else {
-				qu.Debug("万能爬虫验证异常：", err)
+			err := OmnipotentCodeCheck(tmp) //通用验证
+			if err != nil {
+				qu.Debug("通用验证失败")
 			}
+
 		}(r)
 	}
 	wg.Wait()
 }
 
-func OmnipotentCodeCheck(url, stype string) (result map[string]interface{}, err error) {
+// OmnipotentCodeCheck 通用验证
+func OmnipotentCodeCheck(tmp map[string]interface{}) (err error) {
+	defer qu.Catch()
+	url := qu.ObjToString(tmp["href"])
+	stype := qu.ObjToString(tmp["stype"])
 	client := &http.Client{
 		Timeout: 2 * time.Minute,
 	}
@@ -162,6 +99,61 @@ func OmnipotentCodeCheck(url, stype string) (result map[string]interface{}, err
 	if err != nil {
 		return
 	}
+	var result map[string]interface{}
 	err = json.Unmarshal(respBody, &result)
+	if err == nil && result != nil {
+		//保存验证记录
+		result["code"] = tmp["code"]
+		result["site"] = tmp["site"]
+		result["channel"] = tmp["channel"]
+		result["comeintime"] = time.Now().Unix()
+		result["href"] = url
+		util.MgoEB.Save("luaconfig_omnipotent_log", result) //保存ai识别记录
+		omnipotent, _ := result["omnipotent"].(bool)
+		if omnipotent { //验证成功，更新lua信息
+			set := map[string]interface{}{
+				"autotype":    1,
+				"cssmark":     result["cssmark"],
+				"recovertime": time.Now().Unix(),
+				"claimtime":   time.Now().Unix(),
+				"claimtype":   2, //已认领
+				"state":       11,
+				//默认机器人用户
+				"createuser":      UserName,
+				"createuserid":    UserId,
+				"createuseremail": UserEmail,
+				"modifyuser":      UserName,
+				"modifyuserid":    UserId,
+			}
+			//只更新未认领状态的爬虫
+			util.MgoEB.Update("luaconfig",
+				map[string]interface{}{
+					"code":      tmp["code"],
+					"claimtype": 0,
+				},
+				map[string]interface{}{
+					"$set": set,
+				},
+				false, false)
+			time.Sleep(5 * time.Second)
+			one, _ := util.MgoEB.FindOneByField("luaconfig",
+				map[string]interface{}{
+					"code":      tmp["code"],
+					"claimtype": 2,
+				},
+				nil,
+			)
+			if len(*one) > 0 { //保存线上表
+				util.MgoEB.SaveByOriID("luaconfig_online", *one)
+				err = CommCodeMgr(*one)
+				if err != nil {
+					qu.Debug("爬虫发布失败：", err.Error())
+				}
+			} else {
+				qu.Debug("爬虫查询异常，发布失败：", tmp["code"])
+			}
+		}
+	}
 	return
 }
+*/

src/config.json

@@ -50,7 +50,6 @@
         "7700": "comm"
     },
     "serveraddress": "127.0.0.1:8030",
-    "jsserveraddress":  "127.0.0.1:8031",
     "word":{
     	"keyword":"(抽签|中标|招标|成交|合同|中标候选人|资格预审|拟建|邀请|询价|比选|议价|竞价|磋商|采购|招投标|答疑|变更公告|更正公告|竞争性谈判|竞谈|意见征询|澄清|单一来源|流标|废标|验收公告|中止|终止|违规|处罚|征集公告|开标结果|评审结果|监理|招租|租赁|评判结果|项目|遴选|补遗|竞标|征求意见|标段|定点结果|项目评审公示|采购项目违规|采购活动中违规|项目行政处罚|采购行政处罚|项目审批公示)",
     	"notkeyword":"(招聘|拍卖|出租|出让|使用权|资产)"
@@ -105,9 +104,11 @@
     "renderaddr": "http://192.168.3.182:8050/render.json",
     "proxyaddr": "http://cc.spdata.jianyu360.com/crawl/proxy/socks5/fetch",
     "proxyauthor": "Basic amlhbnl1MDAxOjEyM3F3ZSFB",
-    "optcodeserver": "http://127.0.0.1:7081/check",
+    "optcodeserver": "http://127.0.0.1:7081/aicheck",
     "codedbserever": "https://jsspider.jydev.jianyu360.com:18080",
     "taskmgrserever": "https://jsspider.jydev.jianyu360.com:18082",
+    "checkserver": "http://jsspider.jydev.jianyu360.com:1107/checkserver/save",
+    "rosedbserver": "http://jsspider.jydev.jianyu360.com:18084",
     "luadisablelib": {
         "baselib": {
             "print": false

src/front/claim.go

@@ -183,6 +183,7 @@ func (f *Front) ReturnCode() {
 
 func UpdateCodeAndSaveLog(lua []map[string]interface{}, f *Front) {
 	update := [][]map[string]interface{}{}
+	updateTask := [][]map[string]interface{}{}
 	save := []map[string]interface{}{}
 	for _, l := range lua {
 		loginuser := f.GetSession("loginuser")
@@ -201,11 +202,25 @@ func UpdateCodeAndSaveLog(lua []map[string]interface{}, f *Front) {
 				"modifyuser":      loginuser,
 				"modifyuserid":    userid,
 				"claimtime":       now,
-				"state":           Sp_state_0, //通用平台退回爬虫状态为：无法标注12
+				"state":           Sp_state_0, //可视化平台退回爬虫状态为：无法标注12
 				"recovertime":     recovertime,
 			}},
 		}
+		//认领爬虫时，将任务一并带走
+		upTask := []map[string]interface{}{
+			{
+				"s_code": l["code"],
+				"i_state": map[string]interface{}{
+					"$lte": 3,
+				},
+			},
+			{"$set": map[string]interface{}{
+				"s_modify":   loginuser,
+				"s_modifyid": userid,
+			}},
+		}
 		update = append(update, up)
+		updateTask = append(updateTask, upTask)
 		save = append(save, map[string]interface{}{
 			"site":             l["site"],
 			"code":             l["code"],
@@ -219,14 +234,22 @@ func UpdateCodeAndSaveLog(lua []map[string]interface{}, f *Front) {
 			"returntime":       int64(0),
 			"important":        spiderimportant,
 			"returnreason":     "",
-			"claimrecovertype": 1,
+			"claimrecovertype": 1, //主动认领
 			"source":           "golua平台爬虫认领",
 		})
 	}
 	//更新爬虫信息
-	MgoEB.UpdateBulk("luaconfig", update...)
+	if len(update) > 0 {
+		MgoEB.UpdateBulk("luaconfig", update...)
+	}
+	//更新任务信息
+	if len(updateTask) > 0 {
+		MgoEB.UpdateBulk("task", updateTask...)
+	}
 	//保存认领日志
-	MgoEB.SaveBulk("lua_logs_claim", save...)
+	if len(save) > 0 {
+		MgoEB.SaveBulk("lua_logs_claim", save...)
+	}
 }
 
 // CreateRecovertime 生成归还时间

src/front/front.go

@@ -97,12 +97,12 @@ type Front struct {
 	claimCode  xweb.Mapper `xweb:"/center/claim/claimcode"`  //爬虫认领
 	returnCode xweb.Mapper `xweb:"/center/claim/returncode"` //爬虫归还
 
-	//通用爬虫
-	commCodesCenter xweb.Mapper `xweb:"/center/commMonitor"`               //通用爬虫管理中心
+	//可视化爬虫
+	commCodesCenter xweb.Mapper `xweb:"/center/commMonitor"`               //可视化爬虫管理中心
 	updateCodeBase  xweb.Mapper `xweb:"/center/commspider/updatecodebase"` //
 }
 
-const Sp_state_0, Sp_state_1, Sp_state_2, Sp_state_3, Sp_state_4, Sp_state_5, Sp_state_6, Sp_state_7, Sp_state_8, Sp_state_9, Sp_state_10 = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 //0待完成，1待审核，2打回，3发布，4作废，5已上架，6已下架，7无发布，8需登录，9转python，10已删除
+//const Sp_state_0, Sp_state_1, Sp_state_2, Sp_state_3, Sp_state_4, Sp_state_5, Sp_state_6, Sp_state_7, Sp_state_8, Sp_state_9, Sp_state_10 = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 //0待完成，1待审核，2打回，3发布，4作废，5已上架，6已下架，7无发布，8需登录，9转python，10已删除
 
 var spinfos sync.Map = sync.Map{}
 var SessMap map[string]*httpsession.Session
@@ -193,7 +193,7 @@ func (f *Front) LoadIndex() {
 			if state > -1 {
 				query["state"] = state
 			} else if auth == u.Role_Examine {
-				query["state"] = Sp_state_1
+				query["state"] = u.Sp_state_1
 			}
 		}
 		sort := `{"%s":%d}`
@@ -341,7 +341,7 @@ func (f *Front) ImportLua() {
 						href := cells[4].Value
 						one, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"href": href})
 						if len(*one) > 0 {
-							errorinfo[cells[1].Value] = "第" + strconv.Itoa(k) + "行爬虫" + cells[1].Value + ",该栏目已存在,请确认"
+							errorinfo[code] = "第" + strconv.Itoa(k) + "行爬虫" + code + ",该栏目已存在,请确认"
 							//continue
 						}
 						o["channeladdr"] = href
@@ -372,13 +372,13 @@ func (f *Front) ImportLua() {
 						//爬虫类型
 						infoformat, _ := cells[16].Int()
 						if infoformat < 1 {
-							errorinfo[cells[1].Value] = "第" + strconv.Itoa(k) + "行爬虫" + cells[1].Value + ",缺少爬虫类型信息"
+							errorinfo[code] = "第" + strconv.Itoa(k) + "行爬虫" + code + ",缺少爬虫类型信息"
 							continue
 						}
 						o["infoformat"] = infoformat
 
 						if strings.Contains(code, "bidding") {
-							errorinfo[cells[1].Value] = "第" + strconv.Itoa(k) + "行爬虫" + cells[1].Value + ",存储表错误"
+							errorinfo[code] = "第" + strconv.Itoa(k) + "行爬虫" + code + ",存储表错误"
 							continue
 						}
 						o["coll"] = coll
@@ -387,7 +387,7 @@ func (f *Front) ImportLua() {
 						//o["transfercode"] = qu.IntAll(Transfercode[table])
 						ok := saveLua(o) //保存爬虫
 						if ok == false {
-							errorinfo[cells[1].Value] = "第" + strconv.Itoa(k) + "行找不到作者,已经过滤"
+							errorinfo[code] = "第" + strconv.Itoa(k) + "行找不到作者,已经过滤"
 						} else {
 							o["status"] = 1
 							o["author"] = o["modifyuser"]
@@ -416,6 +416,8 @@ func saveLua(o map[string]interface{}) bool {
 	platform := qu.ObjToString(o["platform"])
 	var userid, email string
 	claimLog := map[string]interface{}{}
+	now := time.Now().Unix()
+	expiretime := now + 5*86400
 	if modifyuser != "" { //指定维护人
 		one, _ := u.MgoEB.FindOne("user", map[string]interface{}{"s_name": modifyuser, "i_auth": 1, "i_delete": 0})
 		if len(*one) == 0 {
@@ -423,37 +425,33 @@ func saveLua(o map[string]interface{}) bool {
 		}
 		email = qu.ObjToString((*one)["s_email"])
 		userid = mongodb.BsonIdToSId((*one)["_id"])
-		if platform != "python" {
-			now := time.Now().Unix()
-			recovertime := now + 365*86400
-			param["claimtime"] = now //认领时间
-			param["claimtype"] = CLAIMTYPECLAIMED
-			param["recovertime"] = recovertime
-			claimLog = map[string]interface{}{
-				"site":             o["name"],
-				"code":             o["code"],
-				"channel":          o["channel"],
-				"modifyuser":       modifyuser,
-				"priority":         priority,
-				"stype":            "认领",
-				"comeintime":       now,
-				"claimtime":        now,
-				"recovertime":      recovertime,
-				"returntime":       int64(0),
-				"important":        false,
-				"returnreason":     "",
-				"claimrecovertype": 0,
-				"source":           "爬虫指定维护人导入",
-			}
-		} else {
-			param["recovertime"] = int64(0)       //回收时间
-			param["claimtime"] = int64(0)         //认领时间
-			param["claimtype"] = CLAIMTYPEHISTORY //爬虫认领状态（历史爬虫）
+		recovertime := u.GetRecoverTime(now, platform)
+		param["claimtime"] = now              //认领时间
+		param["claimtype"] = CLAIMTYPECLAIMED //认领状态
+		param["recovertime"] = recovertime    //回收时间
+		param["expiretime"] = expiretime      //过期时间
+		claimLog = map[string]interface{}{
+			"site":             o["name"],
+			"code":             o["code"],
+			"channel":          o["channel"],
+			"modifyuser":       modifyuser,
+			"priority":         priority,
+			"stype":            "认领",
+			"comeintime":       now,
+			"claimtime":        now,
+			"recovertime":      recovertime,
+			"expiretime":       expiretime,
+			"returntime":       int64(0),
+			"important":        false,
+			"returnreason":     "",
+			"claimrecovertype": 0,
+			"source":           "爬虫指定维护人导入",
 		}
 	} else { //未指定人
-		param["recovertime"] = int64(0)         //回收时间
 		param["claimtime"] = int64(0)           //认领时间
 		param["claimtype"] = CLAIMTYPEUNCLAIMED //爬虫认领状态（未认领）
+		param["recovertime"] = int64(0)         //回收时间
+		param["expiretime"] = expiretime        //过期时间
 	}
 	common := []interface{}{
 		AutoTpl["Base.SpiderCode"],
@@ -560,6 +558,8 @@ func saveLua(o map[string]interface{}) bool {
 	param["grade"] = 0                //爬虫难易度（主要用于python爬虫使用）
 	param["spiderimportant"] = false  //是否为重点网站爬虫
 	param["urgency"] = 0
+	param["autotype"] = 0      //自动化采集类型（1：通用；2：整站）
+	param["allowclaim"] = true //允许被其它平台认领开发
 	//qu.Debug("param---", param)
 
 	ok := spider.SaveSpider(o["code"].(string), param)
@@ -672,9 +672,11 @@ func (f *Front) ImportAi() {
 						code = u.SymbolReg.ReplaceAllString(code, "")
 						code = u.CheckCode(code)
 						o := make(map[string]interface{})
-						o["name"] = u.SymbolReg.ReplaceAllString(cells[0].Value, "")
+						site := u.SymbolReg.ReplaceAllString(cells[0].Value, "")
+						o["name"] = site
 						o["code"] = u.SymbolReg.ReplaceAllString(code, "")
-						o["channel"] = cells[2].Value
+						channel := cells[2].Value
+						o["channel"] = channel
 						spiderremark := cells[3].Value
 						if spiderremark == "" {
 							spiderremark = `采集“` + cells[2].Value + `”栏目（含子栏目）`
@@ -710,34 +712,45 @@ func (f *Front) ImportAi() {
 						//爬虫类型
 						infoformat, _ := cells[16].Int()
 						if infoformat < 1 {
-							errorinfo[cells[1].Value] = "第" + strconv.Itoa(k) + "行爬虫" + cells[1].Value + ",缺少爬虫类型信息"
+							errorinfo[code] = "第" + strconv.Itoa(k) + "行爬虫" + code + ",缺少爬虫类型信息"
 							continue
 						}
 						o["infoformat"] = infoformat
 						if strings.Contains(code, "bidding") {
-							errorinfo[cells[1].Value] = "第" + strconv.Itoa(k) + "行爬虫" + cells[1].Value + ",存储表错误"
+							errorinfo[code] = "第" + strconv.Itoa(k) + "行爬虫" + code + ",存储表错误"
 							continue
 						}
 						o["coll"] = coll
 						ok := saveLuaAi(o) //保存爬虫
 						if ok == false {
-							errorinfo[cells[1].Value] = "第" + strconv.Itoa(k) + "行找不到作者,已经过滤"
+							errorinfo[code] = "第" + strconv.Itoa(k) + "行找不到作者,已经过滤"
 						} else {
 							o["status"] = 1
 							o["author"] = o["modifyuser"]
 							o["importuser"] = f.GetSession("username")
 							u.MgoEB.Save("import", o)
-							stype := "bid"
-							if infoformat == 5 {
-								stype = "news"
-							}
-							commutil.CommCodesCache <- map[string]interface{}{
-								"code":    code,
-								"href":    href,
-								"site":    o["name"],
-								"channel": o["channel"],
-								"stype":   stype,
+							//Ai通用校验
+							err := commutil.AiCheckTaskPub(site, channel, code, href, coll, infoformat, true)
+							if err != nil {
+								errorinfo[code] = "第" + strconv.Itoa(k) + "行爬虫" + code + ",任务发布失败，请联系管理员！"
+								qu.Debug("任务发布失败：", code, err)
+							} else {
+								qu.Debug("worker任务发布成功：", code)
 							}
+							//stype := "bid"
+							//if infoformat == 5 {
+							//	stype = "news"
+							//}
+							//commutil.CommCodesCache <- map[string]interface{}{
+							//	"code":          code,
+							//	"href":          href,
+							//	"site":          o["name"],
+							//	"channel":       o["channel"],
+							//	"stype":         stype,
+							//	"infoformat":    infoformat,
+							//	"coll":          coll,
+							//	"spidercompete": true,
+							//}
 						}
 					}
 				}
@@ -771,14 +784,15 @@ func saveLuaAi(o map[string]interface{}) bool {
 		"spidertype":   "history",
 		"spiderremark": o["spiderremark"],
 		"omnipotent":   false,
+		"autotype":     0, //自动化采集类型（1：通用；2：整站）
+		"allowclaim":   true,
 	}
 	AutoTpl["Base.SpiderName"] = o["name"]
 	AutoTpl["Base.SpiderCode"] = o["code"]
 	AutoTpl["Base.SpiderChannel"] = o["channel"]
 	AutoTpl["Base.SpiderTargetChannelUrl"] = o["channeladdr"]
-	modifyuser := o["modifyuser"].(string)
-	var userid, email string
-	claimLog := map[string]interface{}{}
+	//modifyuser := o["modifyuser"].(string)
+	//var userid, email string
 	common := []interface{}{
 		AutoTpl["Base.SpiderCode"],
 		AutoTpl["Base.SpiderName"],
@@ -833,11 +847,11 @@ func saveLuaAi(o map[string]interface{}) bool {
 	param["site"] = o["name"]
 	param["href"] = o["channeladdr"]
 	param["channel"] = o["channel"]
-	param["createuser"] = modifyuser
-	param["createuserid"] = userid
-	param["createuseremail"] = email
-	param["modifyuser"] = modifyuser
-	param["modifyuserid"] = userid
+	param["createuser"] = u.AI_USER
+	param["createuserid"] = u.AI_USERID
+	param["createuseremail"] = u.AI_USEREMAIL
+	param["modifyuser"] = u.AI_USER
+	param["modifyuserid"] = u.AI_USERID
 	param["modifytime"] = time.Now().Unix()
 	param["state"] = 0 //未完成
 	if qu.IntAll(o["event"]) > 0 {
@@ -850,7 +864,7 @@ func saveLuaAi(o map[string]interface{}) bool {
 		model[k] = qu.ObjToString(o[k])
 	}
 	param["model"] = model
-	param["next"] = email
+	param["next"] = u.AI_USEREMAIL
 	incrementevent := qu.ObjToString(o["incrementevent"])
 	if movevent, ok := util.Config.Uploadevents[incrementevent].(string); ok && movevent != "" {
 		param["spidermovevent"] = movevent
@@ -931,10 +945,6 @@ func saveLuaAi(o map[string]interface{}) bool {
 				}}, true, false)
 			}
 		}
-		//生成认领日志
-		if len(claimLog) > 0 {
-			u.MgoEB.Save("lua_logs_claim", claimLog)
-		}
 		//爬虫基本属性
 		u.MgoEB.Save("luaconfig_cfg", map[string]interface{}{
 			"site_toptype":   site_toptype,
@@ -1160,7 +1170,7 @@ func (f *Front) SpiderUpdatePlatform() {
 		}
 		set := map[string]interface{}{}
 		set["platform"] = platform
-		if platform == "python" && platform == "jschrome" {
+		if platform != "golua平台" && platform != "chrome" {
 			var err error
 			var b bool
 			b = u.MgoS.Update("spider_heart", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"del": true}}, false, true)
@@ -1405,13 +1415,12 @@ func (f *Front) UpdateESP() {
 	} else if w == "platform" {
 		//常规字段更新
 		set["platform"] = val
-		//set["comeintime"] = time.Now().Unix()
-		if val == "python" && val == "jschrome" {
+		if val != "golua平台" && val != "chrome" {
 			b := u.MgoS.Update("spider_heart", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"del": true}}, false, true)
 			qu.Debug("Del Heart:", b)
 			var err error
 			pf := qu.ObjToString((*one)["platform"])
-			if pf == "golua平台" || pf == "chrome" { //爬虫所属golua平台
+			if pf == "golua平台" || pf == "chrome" { //爬虫原平台所属golua平台，进行下架
 				b, err = spider.UpdateSpiderByCodeState(code, "6", qu.IntAll((*one)["event"])) //下架
 				if b && err == nil {
 					//历史节点下架为了避免线上运行爬虫待完成时改为7000采集历史，但是又转到其他平台，导致原线上运行节点爬虫并未下线，心跳异常
@@ -1464,7 +1473,7 @@ func (f *Front) UpdateESP() {
 		set["event"] = event
 		set["incrementevent"] = event
 		set["state"] = 6
-		b, err := UpStateAndUpSpider(code, "", "", "", Sp_state_6) //线上爬虫下架
+		b, err := UpStateAndUpSpider(code, "", "", "", u.Sp_state_6) //线上爬虫下架
 		qu.Debug("爬虫下架成功：", b)
 		if !b || err != nil {
 			f.Write("n")

src/front/spider.go

@@ -415,7 +415,7 @@ func (f *Front) SaveStep() {
 				param["code"] = (*one)["code"]
 				//开发员关联任务修改爬虫状态
 				state = qu.IntAll((*one)["state"])
-				if auth == u.Role_Dev && state >= Sp_state_3 && restate != 1 { //开发员修改,已经审核通过(不包含已上架),状态重置为待完成(restate!=1判断，重采修改保存爬虫时不修改爬虫状态)
+				if auth == u.Role_Dev && state >= u.Sp_state_3 && restate != 1 { //开发员修改,已经审核通过(不包含已上架),状态重置为待完成(restate!=1判断，重采修改保存爬虫时不修改爬虫状态)
 					param["state"] = 0
 				} else {
 					param["state"] = state
@@ -1451,14 +1451,14 @@ func (f *Front) BatchShelves() {
 	errCode := []string{}
 	var err error
 	b := false
-	if IsHasUpState(auth, Sp_state_5) {
+	if IsHasUpState(auth, u.Sp_state_5) {
 		if state == 5 { //批量上架
 			for _, code := range codes {
 				if _, ok := u.CodeTimeCache.Get(code); ok { //避免短时间内重复上架
 					errCode = append(errCode, code+"重复上架")
 				} else {
 					u.CodeTimeCache.Set(code, "", time.Minute*2) //上架时，内存记录上架爬虫
-					_, err = UpStateAndUpSpider(code, "", "", "", Sp_state_5)
+					_, err = UpStateAndUpSpider(code, "", "", "", u.Sp_state_5)
 					if err != nil {
 						errCode = append(errCode, code)
 					} else {
@@ -1468,7 +1468,7 @@ func (f *Front) BatchShelves() {
 			}
 		} else { //批量下架
 			for _, code := range codes {
-				b, err = UpStateAndUpSpider(code, "", "", "", Sp_state_6)
+				b, err = UpStateAndUpSpider(code, "", "", "", u.Sp_state_6)
 				if !b || err != nil {
 					errCode = append(errCode, code)
 				}
@@ -1489,7 +1489,7 @@ func (f *Front) BatchShelves() {
 func (f *Front) UpState() error {
 	username := f.GetSession("username").(string)
 	code := f.GetString("code")
-	state, _ := f.GetInt("state")
+	state, _ := f.GetInteger("state")
 	id := f.GetString("taskId")
 	reason := f.GetString("reason")
 	auth := qu.IntAll(f.GetSession("auth"))
@@ -1512,34 +1512,34 @@ func (f *Front) UpState() error {
 		xgTime = xgTimeTmp.Unix()
 	}
 	f.DelSession(id)
-	if IsHasUpState(auth, int(state)) {
-		b, err := UpStateAndUpSpider(code, "", reason, username, int(state)) //更新爬虫状态
-		if b && state == Sp_state_1 {                                        //提交审核
+	if IsHasUpState(auth, state) {
+		b, err := UpStateAndUpSpider(code, "", reason, username, state) //更新爬虫状态
+		if b && state == u.Sp_state_1 {                                 //提交审核
 			//有对应任务跳转提交记录页
 			taskid = CheckTask(codeArr, 1)
 			if len(taskid) > 0 {
 				res["istotask"] = true
 				res["taskid"] = taskid[0]
 			}
-		} else if b && state == Sp_state_2 { //打回
+		} else if b && state == u.Sp_state_2 { //打回
 			taskid = CheckTask(codeArr, 2)
 			if len(taskid) > 0 {
 				//UpTaskState([]string{taskid}, 2)     //修改状态
 				UpTaskState(taskid, 2, "", int64(0)) //修改任务状态
 				SaveRemark(taskid, reason, username) //保存记录信息
 			}
-		} else if b && state == Sp_state_3 { //审核通过
+		} else if b && state == u.Sp_state_3 { //审核通过
 			taskid = CheckTask(codeArr, 3)
 			if len(taskid) > 0 {
 				//UpTaskState([]string{taskid}, 3)
 				UpTaskState(taskid, 3, "", int64(0))
 				SaveRemark(taskid, "", username)
 			}
-		} else if b && state == Sp_state_6 { //下架
+		} else if b && state == u.Sp_state_6 { //下架
 			//下架成功删除心跳数据
 			flag := DelSpiderHeart(code)
 			log.Println(code, "---下架删除download数据：", flag)
-		} else if b && state == Sp_state_7 { //反馈
+		} else if b && state == u.Sp_state_7 { //反馈
 			taskid = CheckTask(codeArr, 7)
 			if len(taskid) > 0 {
 				UpTaskState(taskid, 7, reason, xgTime)
@@ -1587,16 +1587,16 @@ func UpStateAndUpSpider(code, id, reason, username string, state int) (bool, err
 		}
 		//oldstate := qu.IntAll(one["state"])
 		switch state {
-		case Sp_state_4, Sp_state_6: //作废、下架
+		case u.Sp_state_4, u.Sp_state_6: //作废、下架
 			// if oldstate == Sp_state_5 {
 			// 	upresult = false
 			// 	err = errors.New("已上架不允许作废")
 			// } else {
 			// 	upresult = true
 			// }
-			upresult, err = spider.UpdateSpiderByCodeState(code, fmt.Sprint(state), event) //下架
+			upresult, err = spider.UpdateSpiderByCodeState(code, "6", event) //下架
 			qu.Debug("下架:", upresult, code)
-		case Sp_state_5: //上架(爬虫端在更新上架的时候为了更新内存中字段，采用先下架上架)
+		case u.Sp_state_5: //上架(爬虫端在更新上架的时候为了更新内存中字段，采用先下架上架)
 			if downevent := qu.IntAll((*one)["downevent"]); downevent != 0 { //爬虫开发修改爬虫节点，审核人员上架爬虫时，原来爬虫所在节点下架
 				upresult, err = spider.UpdateSpiderByCodeState(code, "6", downevent)
 				qu.Debug(code, "下架历史节点：", downevent)
@@ -1611,7 +1611,7 @@ func UpStateAndUpSpider(code, id, reason, username string, state int) (bool, err
 				upresult, err = spider.UpdateSpiderByCodeState(code, fmt.Sprint(state), event)
 				qu.Debug("上架:", upresult, code, event)
 			}
-		case Sp_state_3: //审核通过
+		case u.Sp_state_3: //审核通过
 			//校验爬虫三级页是否有附件下载方法
 			str_content := qu.ObjToString((*one)["str_content"])
 			if !strings.Contains(str_content, "downloadFile") && !strings.Contains(str_content, "getFileAttachmentsArrayWithTag") {
@@ -1640,14 +1640,14 @@ func UpStateAndUpSpider(code, id, reason, username string, state int) (bool, err
 				}
 				upresult = u.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, up, true, false)
 			} else {
-				if state == Sp_state_1 { //提交审核
+				if state == u.Sp_state_1 { //提交审核
 					upset["l_complete"] = time.Now().Unix()
 					upset["report"] = ""
-				} else if state == Sp_state_7 { //反馈问题
+				} else if state == u.Sp_state_7 { //反馈问题
 					upset["report"] = reason
 					upset["state"] = 1 //反馈后爬虫改为待审核
 					upset["l_complete"] = time.Now().Unix()
-				} else if state == Sp_state_3 { //审核通过
+				} else if state == u.Sp_state_3 { //审核通过
 					if (*one)["event"] == nil {
 						upset["event"] = event
 						//upset["modifytime"] = time.Now().Unix()
@@ -1656,14 +1656,14 @@ func UpStateAndUpSpider(code, id, reason, username string, state int) (bool, err
 					upset["l_uploadtime"] = time.Now().Unix()
 					UpdateSiteTask(code)                       //更新重点站点任务
 					UpdateLuaClaimtype(code, CLAIMTYPEHISTORY) //更新爬虫认领状态
-				} else if state == Sp_state_2 { //打回原因
+				} else if state == u.Sp_state_2 { //打回原因
 					upset["reason"] = reason
 					//清理机检结果
 					unset["check"] = ""
 					unset["checkok"] = ""
 					unset["checktime"] = ""
 					LuaCheckSaveLog(one) //保存机检日志
-				} else if state == Sp_state_5 { //上架
+				} else if state == u.Sp_state_5 { //上架
 					upset["l_checktime"] = 0 //核对时间重置
 					//清理机检结果
 					unset["check"] = ""
@@ -1672,7 +1672,7 @@ func UpStateAndUpSpider(code, id, reason, username string, state int) (bool, err
 					LuaCheckSaveLog(one) //保存机检日志
 					LuaSaveLog(code, username, one, 1)
 					go ModifyLogs_SaveCodeLogs(code, *one)
-				} else if state == Sp_state_4 { //作废，作废原因
+				} else if state == u.Sp_state_4 { //作废，作废原因
 					upset["disablereason"] = reason
 					//upset["modifytime"] = time.Now().Unix()
 					upset["l_uploadtime"] = time.Now().Unix() //l_complete爬虫完成时间
@@ -1686,9 +1686,9 @@ func UpStateAndUpSpider(code, id, reason, username string, state int) (bool, err
 				//upresult = u.MgoE.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": upset}, false, false)
 				upresult = u.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, up, false, false)
 				qu.Debug("提交日志：", code, upset, upresult)
-				if upresult && (state == Sp_state_2 || state == Sp_state_3) { //打回、审核记录日志
+				if upresult && (state == u.Sp_state_2 || state == u.Sp_state_3) { //打回、审核记录日志
 					types := "打回"
-					if state == Sp_state_3 {
+					if state == u.Sp_state_3 {
 						types = "审核"
 					}
 					event := qu.IntAll((*one)["event"])
@@ -1706,7 +1706,7 @@ func UpStateAndUpSpider(code, id, reason, username string, state int) (bool, err
 					}
 					if !strings.HasSuffix(code, u.Bu) { //凡是以_bu结尾的爬虫一律不计入审核记录
 						//新爬虫审核记录表
-						if event == 7000 && (state == Sp_state_3 || state == Sp_state_2) {
+						if event == 7000 && (state == u.Sp_state_3 || state == u.Sp_state_2) {
 							count := u.MgoEB.Count("lua_logs_auditor", map[string]interface{}{"code": code, "types": "审核"})
 							if count == 0 { //新爬虫审核记录
 								u.MgoEB.Save("lua_logs_auditor_new", obj)
@@ -1841,13 +1841,13 @@ func (f *Front) ChangeEvent() {
 	info, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
 	if len(*info) > 0 {
 		oldevent := qu.IntAll((*info)["event"])
-		if qu.IntAll((*info)["state"]) == Sp_state_5 {
+		if qu.IntAll((*info)["state"]) == u.Sp_state_5 {
 			//源节点下架
-			_, err := spider.UpdateSpiderByCodeState(code, fmt.Sprint(Sp_state_6), oldevent)
+			_, err := spider.UpdateSpiderByCodeState(code, fmt.Sprint(u.Sp_state_6), oldevent)
 			set := map[string]interface{}{
 				"$set": map[string]interface{}{
 					"event": qu.IntAll(event),
-					"state": Sp_state_6,
+					"state": u.Sp_state_6,
 				},
 			}
 			//u.MgoE.Update("luaconfig", map[string]interface{}{"code": code}, set, true, false)
@@ -1876,11 +1876,11 @@ func IsHasUpState(auth, state int) bool {
 	rep := false
 	switch auth {
 	case u.Role_Dev:
-		if state == Sp_state_1 || state == Sp_state_7 {
+		if state == u.Sp_state_1 || state == u.Sp_state_7 {
 			rep = true
 		}
 	case u.Role_Examine:
-		if state == Sp_state_2 || state == Sp_state_3 {
+		if state == u.Sp_state_2 || state == u.Sp_state_3 {
 			rep = true
 		}
 	case u.Role_Admin:
@@ -1928,7 +1928,7 @@ func (f *Front) MonitorCenter() {
 		} else {
 			q2 = map[string]interface{}{
 				"state": map[string]interface{}{
-					"$in": []int{Sp_state_3, Sp_state_5, Sp_state_6},
+					"$in": []int{u.Sp_state_3, u.Sp_state_5, u.Sp_state_6},
 				},
 			}
 		}

src/front/spider2.go

@@ -64,9 +64,13 @@ func (f *Front) CommCodesCenter() {
 		limit, _ := f.GetInteger("length")
 		draw, _ := f.GetInteger("draw")
 		searchStr := f.GetString("search[value]")
-		platform := f.GetString("platform")
+		//platform := f.GetString("platform")
+		state, _ := f.GetInteger("state")
+		autotype, _ := f.GetInteger("autotype")
 		search := strings.TrimSpace(searchStr)
-		query := map[string]interface{}{}
+		query := map[string]interface{}{
+			"platform": "可视化平台",
+		}
 		//搜索条件
 		if search != "" {
 			query["$or"] = []interface{}{
@@ -74,9 +78,20 @@ func (f *Front) CommCodesCenter() {
 				map[string]interface{}{"site": map[string]interface{}{"$regex": search}},
 			}
 		}
-		if platform != "-1" {
-			query["platform"] = platform
+		if state != -1 {
+			query["state"] = state
+		}
+		if autotype != -1 {
+			query["autotype"] = autotype
+			if autotype == 0 {
+				query["autotype"] = map[string]interface{}{
+					"$lte": 0,
+				}
+			}
 		}
+		//if platform != "-1" {
+		//	query["platform"] = platform
+		//}
 		sort := `{"%s":%d}`
 		orderIndex := f.GetString("order[0][column]")
 		orderName := f.GetString(fmt.Sprintf("columns[%s][data]", orderIndex))
@@ -85,21 +100,39 @@ func (f *Front) CommCodesCenter() {
 			orderType = -1
 		}
 		sort = fmt.Sprintf(sort, orderName, orderType)
-		luaBases, _ := util.MgoEB.Find("luaconfig_cfg", query, sort, nil, false, start, limit)
-		count := util.MgoEB.Count("luaconfig_cfg", query)
-		qu.Debug("query:", query, start, limit, count, len(*luaBases))
-		for _, lb := range *luaBases {
-			l, _ := util.MgoEB.FindOneByField("luaconfig", map[string]interface{}{"code": lb["code"]}, map[string]interface{}{"state": 1, "workmgr": 1})
-			lb["state"] = (*l)["state"]
-			lb["workmgr"] = (*l)["workmgr"]
+		fields := map[string]interface{}{
+			"site":       1,
+			"channel":    1,
+			"href":       1,
+			"code":       1,
+			"state":      1,
+			"workmgr":    1,
+			"autotype":   1,
+			"infoformat": 1,
+			"platform":   1,
+			"priority":   1,
+		}
+
+		luas, _ := util.MgoEB.Find("luaconfig", query, sort, fields, false, start, limit)
+		count := util.MgoEB.Count("luaconfig", query)
+		qu.Debug("query:", query, start, limit, count, len(*luas))
+		/*for _, lua := range *luas {
+			lb, _ := util.MgoEB.FindOneByField("luaconfig_cfg", map[string]interface{}{"code": lua["code"]}, map[string]interface{}{"property_top": 1, "property_third": 1, "property_sub": 1, "site_subtype": 1, "site_toptype": 1})
+			if len(*lb) > 0 {
+				lua["property_top"] = (*lb)["property_top"]
+				lua["property_third"] = (*lb)["property_third"]
+				lua["property_sub"] = (*lb)["property_sub"]
+				lua["site_subtype"] = (*lb)["site_subtype"]
+				lua["site_toptype"] = (*lb)["site_toptype"]
+			}
 			//var wkText string
 			//if workmgr, ok := (*l)["workmgr"].(map[string]interface{}); ok {
 			//	wkByte, _ := json.Marshal(workmgr)
 			//	wkText = string(wkByte)
 			//}
 			//lb["workmgr"] = wkText
-		}
-		f.ServeJson(map[string]interface{}{"draw": draw, "data": luaBases, "recordsFiltered": count, "recordsTotal": count})
+		}*/
+		f.ServeJson(map[string]interface{}{"draw": draw, "data": luas, "recordsFiltered": count, "recordsTotal": count})
 	} else {
 		events := []string{}
 		for k, _ := range sp.Config.Uploadevents {
@@ -119,7 +152,7 @@ func (f *Front) UpdateCodeBase() {
 	work_timeout, _ := f.GetInteger("work_timeout")
 	interval, _ := f.GetInteger("interval")
 	params := f.GetString("params")
-	qu.Debug(code, itype, work_start, work_end, work_timeout, interval, params)
+	//qu.Debug(code, itype, work_start, work_end, work_timeout, interval, params)
 	var msg string
 	set := map[string]interface{}{
 		"workmgr": map[string]interface{}{
@@ -139,8 +172,8 @@ func (f *Front) UpdateCodeBase() {
 			map[string]interface{}{"code": code},
 			map[string]interface{}{"str_list": 0, "str_content": 0},
 		)
-		if len(*one) > 0 { //保存线上表
-			err := commutil.CommCodeMgr(*one)
+		if len(*one) > 0 {
+			err := commutil.CommCodePub(*one, work_start, work_end, work_timeout, interval, params)
 			if err != nil {
 				msg = err.Error()
 			}

src/main.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"commutil"
 	_ "filter"
 	"front"
 	"luacheck"
@@ -102,7 +101,7 @@ func main() {
 	//爬虫质检
 	go luacheck.LuaCheckStart()
 	//万能爬虫校验
-	go commutil.CommCodeCheck()
+	//go commutil.CommCodeCheck()
 	//提供接口，接收其他数据
 	http.HandleFunc("/spider/infos", func(w http.ResponseWriter, req *http.Request) {
 		data := req.FormValue("data")

src/spider/script.go

@@ -15,7 +15,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/shopspring/decimal"
-	gojs "gorunjs/client"
 	"io"
 	"io/ioutil"
 	login "login/client"
@@ -771,14 +770,6 @@ func (s *Script) LoadScript(site *string, downloadnode, script string, isfile ..
 		S.Push(lua.LString(respCookie))
 		return 3
 	}))
-	s.L.SetGlobal("goRunJs", s.L.NewFunction(func(S *lua.LState) int {
-		param := S.ToString(-2) //list or detail
-		step := S.ToString(-1)  //参数
-		result := gojs.GoRunJsGetResult(s.SCode, param, step)
-		qu.Debug("Go Run Js Result:", param, step, result)
-		S.Push(lua.LString(result))
-		return 1
-	}))
 	s.L.SetGlobal("newDownloadFile", s.L.NewFunction(func(S *lua.LState) int {
 		cookie := S.ToString(-1)
 		head := S.ToTable(-2)

src/taskManager/taskManager.go

@@ -409,7 +409,7 @@ func (t *TaskM) SaveNewTask() {
 	complete := t.GetString("complete")
 	stype := t.GetString("stype")
 	comeintime := time.Now().Unix()
-	var ug string = ""
+	//var ug string = ""
 	newTask := make(map[string]interface{})
 	newTask["s_source"] = "人工"
 	newTask["s_type"] = stype
@@ -418,7 +418,6 @@ func (t *TaskM) SaveNewTask() {
 	newTask["s_code"] = code             //代码
 	newTask["i_state"] = 2               //完成状态
 	newTask["l_comeintime"] = comeintime //创建时间
-	//newTask["s_date"] = time.Now().Format("2006-01-02")
 	newTask["i_times"] = 0
 	newTask["i_pendstate"] = 0
 	newTask["l_checktime"] = time.Now().Unix()
@@ -427,40 +426,44 @@ func (t *TaskM) SaveNewTask() {
 	newTask["i_num"] = 0
 	newTask["i_count"] = 0
 	newTask["f_failrate"] = float64(0)
-	queryL := map[string]interface{}{
+	query := map[string]interface{}{
 		"code": code,
 	}
 	//lua := *mgdb.FindOne("luaconfig", queryL)
-	lua, _ := u.MgoEB.FindOne("luaconfig", queryL)
+	lua, _ := u.MgoEB.FindOne("luaconfig", query)
 	if lua != nil {
+		newTask["i_priority"] = (*lua)["priority"]
 		newTask["s_modifyid"] = (*lua)["createuserid"] //维护人员id
 		newTask["i_event"] = (*lua)["event"]           //节点
 	}
 	newTask["s_modify"] = modify     //维护人员
 	newTask["s_descript"] = descript //描述
 	if urgency == "普通" {
-		ug = "1"
+		//ug = "1"
 		newTask["s_urgency"] = "1" //紧急度
 	} else if urgency == "紧急" {
-		ug = "2"
+		//ug = "2"
 		newTask["s_urgency"] = "2"
 	} else if urgency == "非常紧急" {
-		ug = "3"
+		//ug = "3"
 		newTask["s_urgency"] = "3"
 	} else if urgency == "特别紧急" {
-		ug = "4"
+		//ug = "4"
 		newTask["s_urgency"] = "4"
 	}
-
 	//根据紧急度自动生成最终完成时间
-	if complete == "" {
-		//newTask["l_complete"] = ft.LastTime(timeHour) //最迟完成时间
-		newTask["l_complete"] = ft.CompleteTime(ug)
-	} else { //转成时间戳
-		timeDate, err := time.ParseInLocation("2006-01-02 15:04:05", complete, time.Local)
-		if err == nil {
-			newTask["l_complete"] = timeDate.Unix()
-		}
+	//if complete == "" {
+	//	//newTask["l_complete"] = ft.LastTime(timeHour) //最迟完成时间
+	//	newTask["l_complete"] = ft.CompleteTime(ug)
+	//} else { //转成时间戳
+	//	timeDate, err := time.ParseInLocation("2006-01-02 15:04:05", complete, time.Local)
+	//	if err == nil {
+	//		newTask["l_complete"] = timeDate.Unix()
+	//	}
+	//}
+	timeDate, err := time.ParseInLocation("2006-01-02 15:04:05", complete, time.Local)
+	if err == nil {
+		newTask["l_complete"] = timeDate.Unix()
 	}
 	taskid := u.MgoEB.Save("task", newTask)
 	if taskid != "" {
@@ -619,29 +622,78 @@ func (t *TaskM) UpdateTaskState() {
 	if t.GetSession(id) == nil {
 		t.SetSession(id, xgTime)
 	}
-	query := map[string]interface{}{
-		"_id": mongodb.StringTOBsonId(id),
-	}
-	task, _ := u.MgoEB.FindOne("task", query)
-	updateOk := false
-	if len(*task) > 0 {
-		state := qu.IntAll((*task)["i_state"])
-		if state == 1 { //修改任务状态为待处理
-			update := map[string]interface{}{
-				"$set": map[string]interface{}{
-					"i_state": 2,
-				},
+	var msg string
+	var ok bool
+	code := t.GetString("code")
+	platform := t.GetString("platform")
+	//关联未认领的爬虫
+	lua, _ := u.MgoEB.FindOneByField(
+		"luaconfig",
+		map[string]interface{}{
+			"code": code,
+		},
+		map[string]interface{}{
+			"site":            1,
+			"channel":         1,
+			"modifyuser":      1,
+			"priority":        1,
+			"spiderimportant": 1,
+			"platform":        1,
+			"claimtype":       1,
+		})
+	if len(*lua) > 0 {
+		state := u.Sp_state_2
+		pf := qu.ObjToString((*lua)["platform"])
+		if platform == pf { //爬虫和任务所属平台一致
+			claimtype := qu.IntAll((*lua)["claimtype"])
+			if claimtype == 0 { //未认领爬虫，增加认领机制
+				now := time.Now().Unix()
+				recovertime := u.GetRecoverTime(now, pf)
+				claimLog := map[string]interface{}{
+					"site":             (*lua)["site"],
+					"code":             code,
+					"channel":          (*lua)["channel"],
+					"modifyuser":       (*lua)["modifyuser"],
+					"priority":         (*lua)["priority"],
+					"stype":            "认领",
+					"comeintime":       now,
+					"claimtime":        now,
+					"recovertime":      recovertime,
+					"expiretime":       recovertime,
+					"returntime":       int64(0),
+					"important":        (*lua)["important"],
+					"returnreason":     "",
+					"claimrecovertype": 1,
+					"source":           "任务修改认领爬虫",
+				}
+				update := map[string]interface{}{
+					"claimtime":       now,
+					"claimtype":       1,
+					"expiretime":      recovertime,
+					"recovertime":     recovertime,
+					"createuseremail": t.GetSession("email"),
+					"createuser":      t.GetSession("loginuser"),
+					"createuserid":    t.GetSession("userid"),
+					"modifyuser":      t.GetSession("loginuser"),
+					"modifyuserid":    t.GetSession("userid"),
+					"state":           0,
+				}
+				u.MgoEB.Save("lua_logs_claim", claimLog)
+				u.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": update}, false, false)
 			}
-			updateOk = u.MgoEB.Update("task", query, update, false, false) //更新任务状态
-			code := qu.ObjToString((*task)["s_code"])
-			//mgdb.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"ismodify": true}}, false, false)
-			u.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"ismodify": true}}, false, false)
+			ok = true
+			//修改爬虫任务状态
 		} else {
-			updateOk = true
+			msg = "无效任务，自动关闭！"
+			state = u.Sp_state_6
 		}
+		u.MgoEB.UpdateById("task", id, map[string]interface{}{"$set": map[string]interface{}{"i_state": state}})
+	} else {
+		msg = "未找到爬虫！"
 	}
 	t.ServeJson(map[string]interface{}{
-		"state": updateOk,
+		"ok":  ok,
+		"msg": msg,
 	})
 }
 

src/util/code.go

@@ -8,6 +8,8 @@ import (
 	"strconv"
 )
 
+var Sp_state_0, Sp_state_1, Sp_state_2, Sp_state_3, Sp_state_4, Sp_state_5, Sp_state_6, Sp_state_7, Sp_state_8, Sp_state_9, Sp_state_10 = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 //0待完成，1待审核，2打回，3发布，4作废，5已上架，6已下架，7无发布，8需登录，9转python，10已删除
+
 // CheckCode 检查导入爬虫的唯一性
 func CheckCode(code string) string {
 	query := map[string]interface{}{

src/vpsconfig.json

@@ -12,6 +12,12 @@
             "4": "lt-2zecgkiku3ge497fk7ea",
             "8": "lt-2ze9dvtb8f0x7rqw0r51"
           }
+        },
+        "cn-beijing-k": {
+          "vswitchid": "vsw-2ze3dbu66dzrygpz2g94q",
+          "launchtemplateids": {
+            "8": "lt-2ze8hsjfh3irvtdg0dsv"
+          }
         }
       }
     },

src/web/staticres/js/head.js

@@ -473,6 +473,7 @@ function saveNewTask(){
 	$("body").css("padding-right","0");
 	$modify = $("#myModal-createTask #modify").val();
 	$stype = $("#myModal-createTask #stype").val();
+	$complete = $("#myModal-createTask #complete").val();
 	if($stype == "-1"){
 		alert("请选择任务状态！")
 		return
@@ -481,10 +482,12 @@ function saveNewTask(){
 		return
 	}else if(sessionStorage.getItem("coderw") == "w"){
 		return
+	}else if ($complete == ""){
+		alert("请选择最迟完成时间！")
+		return;
 	}else{
 		$("#myModal-createTask").modal("hide");
 		var $urgency = $("input:radio[name='urgency']:checked").val();
-		$complete = $("#myModal-createTask #complete").val();
 		$site = $("#myModal-createTask #site").val().trim().replace(/\s/g,"");
 		$channel = $("#myModal-createTask #channel").val().trim().replace(/\s/g,"");
 		$modify = $("#myModal-createTask #modify").val();

src/web/templates/commcodes.html

@@ -58,7 +58,7 @@
 <div class="content-wrapper">
 	<section class="content-header">
 		<h1>
-			爬虫属性
+			爬虫列表
 			<small>
 			</small>
 		</h1>
@@ -78,14 +78,15 @@
 						<th>栏目</th>
 						<th>爬虫</th>
 						<th>类型</th>
-						<th>平台</th>
+						<!--<th>平台</th>-->
 						<th>状态</th>
+						<th>AI/整站</th>
 						<th>评分</th>
-						<th>站点一级</th>
-						<th>站点二级</th>
-						<th>属性一级</th>
-						<th>属性二级</th>
-						<th>属性三级</th>
+<!--						<th>站点一级</th>-->
+<!--						<th>站点二级</th>-->
+<!--						<th>属性一级</th>-->
+<!--						<th>属性二级</th>-->
+<!--						<th>属性三级</th>-->
 						<th class="hidden-xs">操作</th>
 					</tr>
 					</thead>
@@ -103,7 +104,7 @@
 					"url": "/js/dataTables.chinese.lang"
 				},
 				"columnDefs": [
-					{ "orderable": false, "targets": [0,1,2,3,4,5,7,8,9,10,11] },
+					{ "orderable": false, "targets": [0,1,2,3,4,5,7] },
 				],
 				"order": [[6,"desc"]],
 				"processing":true,
@@ -140,7 +141,7 @@
 								return "新闻";
 							}
 						}},
-					{ "data": "platform"},
+					/*{ "data": "platform"},*/
 					{ "data": "state",render:function (val,a,row){
 							if(val==0){
 								val = "待完成";
@@ -171,15 +172,26 @@
 							}
 							return val;
 						}},
+					{ "data": "autotype",render:function(val,a,row){
+							if (val == 1){
+								return "AI";
+							}else if (val == 2){
+								return "整站";
+							}
+							return "常规";
+						}},
 					{ "data": "priority"},
-					{ "data": "site_toptype"},
+					/*{ "data": "site_toptype"},
 					{ "data": "site_subtype"},
 					{ "data": "property_top"},
 					{ "data": "property_sub"},
-					{ "data": "property_third"},
+					{ "data": "property_third"},*/
 					{ "data": "_id",render:function(val,a,row){
+							if (row.workmgr == undefined){
+								row.workmgr = {};
+							}
 							var dbutton=$(`<button class="btn btn-sm btn-success" disabled>更新发布</button>`);
-							if (row.state == 11 && row.platform == "通用平台"){
+							if (row.state == 11 && row.platform == "可视化平台"){
 								var workmgrText = JSON.stringify(row.workmgr);
 								dbutton=$(`<button class="btn btn-sm btn-success" data-workmgr='${workmgrText}' onclick="publish('${row.code}', this)">更新发布</button>`);
 							}
@@ -192,30 +204,56 @@
 
 				},
 				"fnServerParams": function (e) {
-					var platform = $("#platform").val();
-					//平台
-					if(platform){
-						e.platform = platform;
+					var state=$("#state").val();
+					var autotype = $("#autotype").val();
+					//状态
+					if(state){
+						e.state = state;
 					}else{
-						e.platform="-1";
+						e.state="-1";
+					}
+					//通用
+					if(autotype){
+						e.autotype = autotype;
+					}else{
+						e.autotype="-1";
 					}
 				}
 			});
 			common.menuActive("commMonitor");
 			ttable.on( 'init.dt', function () {
-				//平台
-				var platform="<option value='-1'>全部</option>"
-				platform+="<option value='golua平台'>golua平台</option>";
-				platform+="<option value='chrome'>chrome</option>";
-				platform+="<option value='python'>python</option>";
-				platform+="<option value='通用平台'>通用平台</option>";
-				platform+="<option value='jschrome'>jschrome</option>";
-				var selectPlatform="<div class='form-group'><label for='name'>平台：</label>"+
-						"<select id='platform' onchange='checkclick(this.value)' class='form-control input-sm'>"+
-						platform+
+				var state="<option value='-1'>全部</option>";
+				state +="<option value='0'>待完成</option>";
+				state +="<option value='1'>待审核</option>"+
+						"<option value='2'>未通过</option>"+
+						"<option value='3'>已通过</option>";
+				state +="<option value='4'>已作废</option>"+
+						"<option value='5'>已上架</option>"+
+						"<option value='6'>已下架</option>";
+				state +="<option value='7'>无发布</option>"+
+						"<option value='8'>需登录</option>"+
+						"<option value='9'>转python</option>"+
+						"<option value='10'>已删除</option>"+
+						"<option value='11'>已上线</option>"+
+						"<option value='12'>无法标注</option>";
+				var selectState="<div class='form-group'><label for='name'>状态：</label>"+
+						"<select id='state' onchange='checkclick(this,\"\")' class='form-control input-sm'>"+
+						state+
+						"</select></div>";
+				$("#spider_filter").prepend("&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;");
+				$("#spider_filter").prepend(selectState);
+
+				//通用
+				var autotype ="<option value='-1'>全部</option>"
+				autotype +="<option value='0'>常规</option>";
+				autotype +="<option value='1'>AI</option>";
+				autotype +="<option value='2'>整站</option>";
+				var selectAutotype="<div class='form-group'><label for='name'>通用：</label>"+
+						"<select id='autotype' onchange='checkclick(this.value)' class='form-control input-sm'>"+
+						autotype+
 						"</select></div>"
 				$("#spider_filter").prepend("&nbsp;&nbsp;");
-				$("#spider_filter").prepend(selectPlatform);
+				$("#spider_filter").prepend(selectAutotype);
 				$("#spider_wrapper .col-sm-6").css({width:"100%"})//样式
 			});
 		})

src/web/templates/head.html

@@ -12,7 +12,7 @@
   <!-- Font Awesome -->
 <!--  <link rel="stylesheet" href="https://cdn.staticfile.org/font-awesome/4.2.0/css/font-awesome.min.css">-->
   <!-- Ionicons -->
-  <link rel="stylesheet" href="https://cdn.staticfile.org/ionicons/2.0.1/css/ionicons.css">
+<!--  <link rel="stylesheet" href="https://cdn.staticfile.org/ionicons/2.0.1/css/ionicons.css">-->
   <!-- Theme style -->
   <link rel="stylesheet" href="/css/AdminLTE.min.css">
    <link rel="stylesheet" href="/css/font-awesome.min.css">
@@ -644,7 +644,7 @@
 			  </a>
 			  <ul class="treeview-menu">
 				  <li><a href="/center/luaMonitor"><i class="glyphicon glyphicon-font"></i>Lua爬虫管理</a></li>
-				  <li><a href="/center/commMonitor"><i class="glyphicon glyphicon-bold"></i>通用爬虫管理</a></li>
+				  <li><a href="/center/commMonitor"><i class="glyphicon glyphicon-bold"></i>可视化爬虫管理</a></li>
 			  </ul>
 			</li>
 			<li data="index_pcdr"><a href="/center/importdata"><i class="glyphicon glyphicon-import"></i> <span>爬虫导入</span></a></li>

src/web/templates/index.html

@@ -62,7 +62,7 @@
 									<option value='golua平台'>golua平台</option>
 									<option value='chrome'>chrome</option>
 									<option value='python'>python</option>
-									<option value='通用平台'>通用平台</option>
+									<option value='可视化平台'>可视化平台</option>
 									<option value='jschrome'>jschrome</option>
 								</select>
 							</div>
@@ -274,6 +274,7 @@
 					<th>状态</th>
 					<th>平台</th>
 					<th>优先级</th>
+					<th>AI/整站</th>
 					<th>认领时间</th>
 					<th>回收时间</th>
 <!--					{{if eq (session "platform") "python"}}-->
@@ -298,7 +299,7 @@
 				  "url": "/js/dataTables.chinese.lang"
 			  },
 			"columnDefs": [
-				{ "orderable": false, "targets": [0,2,4,5,6,7,12,13] },
+				{ "orderable": false, "targets": [0,2,4,5,6,7,13,14] },
 				//爬虫类型
 				{"targets":[4], createdCell: function (cell, cellData, rowData, rowIndex, colIndex) {
 					{{if gt (session "auth") 2}}
@@ -435,7 +436,7 @@
 					{{end}}
 				}}
 			],
-			"order": [[10,"desc"]],
+			"order": [[11,"desc"]],
 			"processing":true,
 			"serverSide": true,
 			"searching": true,
@@ -519,6 +520,14 @@
 				}},
 				{ "data": "platform"},
 				{ "data": "priority"},
+				{ "data": "autotype",render:function(val,a,row){
+					if (val == 1){
+						return "AI";
+					}else if (val == 2){
+						return "整站";
+					}
+					return "常规";
+					}},
 				{ "data": "claimtime","width":"6%",render:function(val,a,row){
 					if (val == 0){
 						return val;
@@ -740,7 +749,7 @@
 				"<option value='golua平台'>golua平台</option>"+
 				"<option value='chrome'>chrome</option>"+
 				"<option value='python'>python</option>"+
-				"<option value='通用平台'>通用平台</option>"+
+				"<option value='可视化平台'>可视化平台</option>"+
 				"<option value='jschrome'>jschrome</option>"+
 				"</select></div>";
 			$("#spider_filter").prepend("&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;");
@@ -1052,7 +1061,7 @@
 				"<option value='golua平台'>golua平台</option>"+
 				"<option value='chrome'>chrome</option>"+
 				"<option value='python'>python</option>"+
-				"<option value='通用平台'>通用平台</option>"+
+				"<option value='可视化平台'>可视化平台</option>"+
 				"<option value='jschrome'>jschrome</option>"+
 				"</select>";
 	};

src/web/templates/lualist.html

@@ -255,7 +255,7 @@
                     "<option value='golua平台'>golua平台</option>"+
                     "<option value='chrome'>chrome</option>"+
                     "<option value='python'>python</option>"+
-                    "<option value='通用平台'>通用平台</option>"+
+                    "<option value='可视化平台'>可视化平台</option>"+
                     "<option value='jschrome'>jschrome</option>"+
                     "</select></div>"
                 $("#spider_filter").prepend("&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;");

src/web/templates/mytask.html

@@ -220,7 +220,7 @@
 			platform+="<option value='golua平台'>golua平台</option>";
 			platform+="<option value='chrome'>chrome</option>";
 			platform+="<option value='python'>python</option>";
-			platform+="<option value='通用平台'>通用平台</option>";
+			platform+="<option value='可视化平台'>可视化平台</option>";
 			platform+="<option value='jschrome'>jschrome</option>";
 			var selectPlatform="<div class='form-group'><label for='name'>平台：</label>"+
 					"<select id='platform' onchange='checkclick(this.value)' class='form-control input-sm'>"+

src/web/templates/mytaskedit.html

@@ -224,10 +224,10 @@
 	var site = task["s_site"];
 	var channel = task["s_channel"];
 	var modify = task["s_modify"];
-	var code = task["s_code"];
+	var taskCode = task["s_code"];
 	var complete = task["l_complete"];
 	var urgency = task["s_urgency"];
-	var comeintime = task["l_complete"];
+	var comeintime = task["l_comeintime"];
 	var state = task["i_state"];
 	var descript = task["s_descript"];
 	var modifyid = task["s_modifyid"];
@@ -574,11 +574,14 @@
 		//	sessionStorage.setItem("startTime",startTime);
 		//}
 		//修改任务状态为处理中
-		$.post("/center/task/updateTaskState",{"id":editId},function(r){
-			if(r&&r.state){
+		$.post("/center/task/updateTaskState",{"id":editId,"code":taskCode,"platform":task["s_platform"]},function(r){
+			if(r&&r.ok){
 				window.location.href = "/center/spider/edit/"+encode+"__"+editId;
 			}else{
-				showTip("后台爬虫下架错误！", 2000, function() {});
+				showMsg(r.msg,function(){
+					// window.history.back();
+					window.location.reload();
+				})
 			}
 		})
 	}

src/web/templates/newtasklist.html

@@ -321,7 +321,7 @@
 			platform+="<option value='golua平台'>golua平台</option>";
 			platform+="<option value='chrome'>chrome</option>";
 			platform+="<option value='python'>python</option>";
-			platform+="<option value='通用平台'>通用平台</option>";
+			platform+="<option value='可视化平台'>可视化平台</option>";
 			platform+="<option value='jschrome'>jschrome</option>";
 			var selectPlatform="<div class='form-group'><label for='name'>平台：</label>"+
 					"<select id='platform' onchange='checkclick(this.value)' class='form-control input-sm'>"+

src/web/templates/sitelist.html

@@ -713,7 +713,7 @@ $(function(){
             "<option value='chrome'>chrome</option>"+
             "<option value='python'>python</option>"+
             "<option value='golua平台,python'>golua平台,python</option>"+
-            "<option value='通用平台'>通用平台</option>"+
+            "<option value='可视化平台'>可视化平台</option>"+
             "<option value='jschrome'>jschrome</option>"+
             "</select></div>"
 		$("#sitelist_filter").prepend("&nbsp;&nbsp;");

src/web/templates/task.html

@@ -321,7 +321,7 @@
 			platform+="<option value='golua平台'>golua平台</option>";
 			platform+="<option value='chrome'>chrome</option>";
 			platform+="<option value='python'>python</option>";
-			platform+="<option value='通用平台'>通用平台</option>";
+			platform+="<option value='可视化平台'>可视化平台</option>";
 			platform+="<option value='jschrome'>jschrome</option>";
 			var selectPlatform="<div class='form-group'><label for='name'>平台：</label>"+
 					"<select id='platform' onchange='checkclick(this.value)' class='form-control input-sm'>"+

src/web/templates/taskedit.html

@@ -382,11 +382,11 @@
 	}
 	
 	
-	function reviseReptile(){
-		//修改任务状态
-		$.post("/center/task/updateTaskState",{"code":code},function(){})
-		window.location.href = "/center/spider/edit/"+code+"__";
-	}
+	// function reviseReptile(){
+	// 	//修改任务状态
+	// 	$.post("/center/task/updateTaskState",{"code":code},function(){})
+	// 	window.location.href = "/center/spider/edit/"+code+"__";
+	// }
 	//分配任务
 	function assignTaskRe(id){
 		if(state == 6 || state == 1 || state == 2 ||  state == 3 ||  state == 4 || state == 5){

src/web/templates/user.html

@@ -296,7 +296,7 @@
 					"<option value='-1'>全部</option>"+
 					"<option value='golua平台'>golua平台</option>"+
 					"<option value='python'>python</option>"+
-					"<option value='通用平台'>通用平台</option>"+
+					"<option value='可视化平台'>可视化平台</option>"+
 					"<option value='jschrome'>jschrome</option>"+
 					"</select></div>"
 			$("#user_filter").prepend("&nbsp;&nbsp;");