爬虫导入；审核记录；爬虫迁移

src/front/front.go

@@ -585,6 +585,7 @@ func (f *Front) Importfile() {
 								errorinfo[cells[1].Value] = "第" + strconv.Itoa(k) + "行找不到作者,已经过滤"
 							} else {
 								o["author"] = name
+								o["importuser"] = f.GetSession("username")
 								u.MgoEB.Save("import", o)
 							}
 						}
@@ -740,9 +741,25 @@ func (f *Front) Importdata() {
 		if f.Method() == "GET" {
 			f.Render("import.html")
 		} else {
-			rss, _ := u.MgoEB.Find("import", nil, `{"timestamp": -1}`, nil, false, -1, -1)
+			query := map[string]interface{}{}
+			searchStr := f.GetString("search[value]")
+			search := strings.TrimSpace(searchStr)
+			if search != "" {
+				query["$or"] = []interface{}{
+					map[string]interface{}{"code": map[string]interface{}{"$regex": search}},
+					map[string]interface{}{"name": map[string]interface{}{"$regex": search}},
+				}
+			}
+			start, _ := f.GetInteger("start")
+			limit, _ := f.GetInteger("length")
+			draw, _ := f.GetInteger("draw")
+			data, _ := u.MgoEB.Find("import", query, `{"timestamp": -1}`, nil, false, start, limit)
+			count := u.MgoEB.Count("import", query)
 			f.ServeJson(map[string]interface{}{
-				"data": *rss,
+				"draw":            draw,
+				"data":            data,
+				"recordsFiltered": count,
+				"recordsTotal":    count,
 			})
 		}
 	} else {
@@ -1070,7 +1087,7 @@ func (f *Front) SpiderCopy() {
 		lua, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
 		if len(*lua) > 0 {
 			codeOld := qu.ObjToString((*lua)["code"])
-			codeNew := codeOld + "_bu"
+			codeNew := codeOld + u.Bu
 			//luaTmp := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": codeNew})
 			luaTmp, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": codeNew})
 			if len(*luaTmp) > 0 {

src/front/luamove.go

@@ -83,6 +83,9 @@ func (lm *LuaMove) LuaMove() {
 			count := u.MgoEB.Count("luamovevent", query)
 			qu.Debug("query:", query, sort, count)
 			data, _ := u.MgoEB.Find("luamovevent", query, sort, nil, false, start, limit)
+			for _, d := range *data {
+				d["encode"] = util.Se.Encode2Hex(qu.ObjToString(d["code"]))
+			}
 			lm.ServeJson(map[string]interface{}{
 				"draw":            draw,
 				"data":            data,
@@ -130,7 +133,8 @@ func (lm *LuaMove) LuaMoveByCode() {
 
 func (lm *LuaMove) CloseAll() {
 	defer qu.Catch()
-	ok := u.MgoEB.Update("luamovevent", map[string]interface{}{"state": 0}, map[string]interface{}{"$set": map[string]interface{}{"state": 1, "updatetime": time.Now().Unix()}}, false, true)
+	//ok := u.MgoEB.Update("luamovevent", map[string]interface{}{"state": 0}, map[string]interface{}{"$set": map[string]interface{}{"state": 1, "updatetime": time.Now().Unix()}}, false, true)
+	ok := u.MgoEB.Del("luamovevent", nil)
 	lm.ServeJson(map[string]interface{}{"ok": ok})
 }
 

src/front/spider.go

@@ -202,6 +202,12 @@ func (f *Front) ViewSpider(id string) error {
 				f.T["spidertype"] = (*lua)["spidertype"]
 				f.T["spidermovevent"] = (*lua)["spidermovevent"]
 				f.T["spiderhistorymaxpage"] = (*lua)["spiderhistorymaxpage"]
+				events := []string{}
+				for k, _ := range util.Config.Uploadevents {
+					events = append(events, k)
+				}
+				sort.Strings(events)
+				f.T["events"] = events
 				if (*lua)["oldlua"] != nil {
 					return f.Render("oldedit.html", &f.T)
 				}
@@ -1187,14 +1193,16 @@ func UpStateAndUpSpider(code, id, reason, username string, state int) (bool, err
 						"site":       (*one)["site"],
 						"channel":    (*one)["channel"],
 					}
-					//新爬虫审核记录表
-					if state == Sp_state_3 {
-						count := u.MgoEB.Count("lua_logs_auditor", map[string]interface{}{"code": code, "types": types})
-						if count == 0 { //新爬虫审核记录
-							u.MgoEB.Save("lua_logs_auditor_new", obj)
+					if !strings.HasSuffix(code, u.Bu) { //凡是以_bu结尾的爬虫一律不计入审核记录
+						//新爬虫审核记录表
+						if state == Sp_state_3 || state == Sp_state_2 {
+							count := u.MgoEB.Count("lua_logs_auditor", map[string]interface{}{"code": code, "types": types})
+							if count == 0 { //新爬虫审核记录
+								u.MgoEB.Save("lua_logs_auditor_new", obj)
+							}
 						}
+						u.MgoEB.Save("lua_logs_auditor", obj) //历史维护爬虫审核记录
 					}
-					u.MgoEB.Save("lua_logs_auditor", obj) //历史维护爬虫审核记录
 				}
 			}
 		}

src/main.go

@@ -107,6 +107,11 @@ func main() {
 			front.SpiderMoveEvent(data)
 		}
 	})
-	go http.ListenAndServe(":6011", nil)
+	go func() {
+		err := http.ListenAndServe(":6011", nil)
+		if err != nil {
+			qu.Debug("6011 Error:", err)
+		}
+	}()
 	xweb.Run(":" + util.Config.Webport)
 }

src/taskManager/taskManager.go

@@ -321,8 +321,8 @@ func (t *TaskM) Mytask() {
 			for _, v := range *task {
 				code := fmt.Sprint(v["s_code"])
 				relatecode := ""
-				//lua, _ := u.MgoE.FindOneByField("luaconfig", map[string]interface{}{"code": code, "state": 5}, map[string]interface{}{"relatecode": 1})
-				lua, _ := u.MgoEB.FindOneByField("luaconfig", map[string]interface{}{"code": code, "state": 5}, map[string]interface{}{"relatecode": 1})
+				//查询关联爬虫
+				lua, _ := u.MgoEB.FindOneByField("luaconfig", map[string]interface{}{"code": code + u.Bu, "state": 5}, map[string]interface{}{"relatecode": 1})
 				if len(*lua) > 0 {
 					relatecode = qu.ObjToString((*lua)["relatecode"])
 				}

src/util/util.go

@@ -81,6 +81,7 @@ var (
 		"winnerorder":        true,
 		"bidopendate":        true,
 	}
+	Bu = "_bu" //创建采历史爬虫后缀
 )
 
 func InitMgo() {

src/web/templates/import.html

@@ -28,6 +28,7 @@
 							<tr>
 								<td>网站名称</td>
 								<td>爬虫代码</td>
+								<td>开发</td>
 								<td>作者</td>
 								<td>时间</td>
 							</tr>
@@ -46,21 +47,23 @@
 			"language": {
                "url": "/js/dataTables.chinese.lang"
            },
-			"paging": true,
 			"lengthChange": false,
 			"ajax": {
             	"url": "/center/importdata",
            		"type": "POST"
        		 },
+			"paging": true,
+			"serverSide": true,
 			"searching": true,
-			"ordering": true,
-			"info": false,
+			"ordering": false,
+			"info": true,
 			"autoWidth": true,
-			"aaSorting": [[ 3, "desc" ]] ,
+			//"aaSorting": [[ 3, "desc" ]] ,
 			"columns": [
 	            {"data": "name"},
 				{"data":"code"},
 				{"data":"author"},
+				{"data":"importuser"},
 				{"data":"timestamp",render:function(val,a,row){
 					if(val){
 						val=val*1000;

src/web/templates/luamovelist.html

@@ -6,7 +6,7 @@
 		     <small>
 				  <button class="btn btn-primary btn-sm" onclick='batchCodeMove("")'>批量迁移</button>
 				  <button class="btn btn-primary btn-sm" onclick="batchCodeClose()">批量关闭</button>
-				 <button class="btn btn-primary btn-sm" onclick="CloseAll()">一键关闭</button>
+				 <button class="btn btn-primary btn-sm" onclick="CloseAll()">一键清除</button>
 			 </small>
 		   </h1>
 		   <ol class="breadcrumb">
@@ -85,7 +85,7 @@
 			"searching": true,
 			"ordering": true,
 			"info": true,
-			"autoWidth": true,
+			"autoWidth": false,
 			"columns": [
 				{"data": "code",render:function(val,a,row){
 					return "<input type='checkbox' ismove='"+row.ismove+"' state='"+row.state+"' code='"+row.code+"' movevent='"+row.toevent+"'/>"
@@ -115,17 +115,19 @@
 						return "已处理"
 					}
 				}},
-        		{"data": "_id",width:"70px",render:function(val,a,row){
-					if(row.state == 1){
-						return ""
-					}
+        		{"data": "_id",width:"91px",render:function(val,a,row){
 					var div=$("<div><div class=\"btn-group\"></div></div>")
-					var buttonMove=$('<a type="button" target="_blank" class="btn btn-sm btn-success">迁移</a>');
-					var buttonClose=$('<a type="button" class="btn btn-sm btn-danger">关闭</a>');
-				  	buttonMove.attr("onclick","codeLuaMove('"+row.code+"','move','"+row.toevent+"')");
-					buttonClose.attr("onclick","codeLuaMove('"+row.code+"','close','')");
-					div.find(".btn-group").append(buttonMove);
-					div.find(".btn-group").append(buttonClose);
+					if(row.state != 1){
+						var buttonMove=$('<a type="button" target="_blank" class="btn btn-sm btn-success">迁移</a>');
+						var buttonClose=$('<a type="button" class="btn btn-sm btn-danger">关闭</a>');
+						buttonMove.attr("onclick","codeLuaMove('"+row.code+"','move','"+row.toevent+"')");
+						buttonClose.attr("onclick","codeLuaMove('"+row.code+"','close','')");
+						div.find(".btn-group").append(buttonMove);
+						div.find(".btn-group").append(buttonClose);
+					}
+					var buttonView=$('<a type="button" target="_blank" class="btn btn-sm btn-default">查看</a>');
+					buttonView.attr("href","/center/spider/view/"+row['encode']);
+					div.find(".btn-group").append(buttonView);
 					return div.html();
 				}}
 			],
@@ -267,7 +269,7 @@
   }
 
   function CloseAll() {
-	  showConfirm("确定关闭所有爬虫？", function() {
+	  showConfirm("确定清除所有爬虫？", function() {
 		  $.ajax({
 			  url: "/center/luamove/closeall",
 			  type: "post",