功能修改

src/front/spider.go

@@ -418,36 +418,45 @@ func LuaTextCheck(list, detail string, type_list int) (b bool, msg string) {
 func (f *Front) Assort() {
 	state, _ := f.GetInteger("state")
 	code := f.GetString("code")
-	sql := map[string]interface{}{
-		"$set": map[string]interface{}{
-			"state":      state,
-			"modifytime": time.Now().Unix(),
-		},
-	}
-	queryT := map[string]interface{}{
-		"code": code,
-	}
-	//下架爬虫
-	lua, _ := u.MgoE.FindOne("luaconfig", queryT)
-	upresult, err := spider.UpdateSpiderByCodeState(code, "6", qu.IntAll((*lua)["event"]))
-	qu.Debug("下架爬虫：", code, upresult, err)
-	if upresult && err == nil {
-		//更新爬虫
-		u.MgoE.Update("luaconfig", queryT, sql, false, false)
-		//关闭任务
+	codes := u.SymbolReg.Split(code, -1)
+	success := true
+	msg := ""
+	for _, code := range codes {
 		query := map[string]interface{}{
-			"s_code": code,
+			"code": code,
 		}
-		sql = map[string]interface{}{
-			"$set": map[string]interface{}{
-				"i_state": 6,
-			},
+		//下架爬虫
+		lua, _ := u.MgoE.FindOne("luaconfig", query)
+		upresult, err := spider.UpdateSpiderByCodeState(code, "6", qu.IntAll((*lua)["event"]))
+		qu.Debug("下架爬虫：", code, upresult, err)
+		if upresult && err == nil {
+			//更新爬虫状态
+			update := map[string]interface{}{
+				"$set": map[string]interface{}{
+					"state":      state,
+					"modifytime": time.Now().Unix(),
+				},
+			}
+			u.MgoE.Update("luaconfig", query, update, false, false)
+			//关闭任务
+			query = map[string]interface{}{
+				"s_code": code,
+			}
+			update = map[string]interface{}{
+				"$set": map[string]interface{}{
+					"i_state": 6,
+				},
+			}
+			u.MgoE.Update("task", query, update, false, true)
+			//删除心跳
+			DelSpiderHeart(code)
+		} else {
+			success = false
+			msg += code + ";"
 		}
-		u.MgoE.Update("task", query, sql, false, true)
-		//删除心跳
-		DelSpiderHeart(code)
 	}
-	f.ServeJson(map[string]interface{}{"upresult": upresult})
+
+	f.ServeJson(map[string]interface{}{"success": success, "msg": msg})
 }
 
 //方法测试
@@ -735,33 +744,46 @@ func (f *Front) SpiderPass() {
 	f.ServeJson(steps)
 }
 
-func (f *Front) DownSpider(id string) {
-	//auth := qu.IntAll(f.GetSession("auth"))
-	//if auth > role_dev {
-	one, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": id})
-	createuserid := qu.ObjToString((*one)["createuserid"])
+func (f *Front) DownSpider(code string) {
+	auth := qu.IntAll(f.GetSession("auth"))
+	user := f.GetSession("loginuser")
+	success := false
 	script := ""
-	filename := id + ".lua"
-	if len(*one) > 0 {
-		if (*one)["oldlua"] != nil {
-			if (*one)["luacontent"] != nil {
-				script = (*one)["luacontent"].(string)
+	if auth > role_dev {
+		success = true
+		one, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": code})
+		createuserid := qu.ObjToString((*one)["createuserid"])
+		filename := code + ".lua"
+		if len(*one) > 0 {
+			if (*one)["oldlua"] != nil {
+				if (*one)["luacontent"] != nil {
+					script = (*one)["luacontent"].(string)
+				}
+			} else {
+				user, _ := u.MgoE.FindById("user", createuserid, nil)
+				name := (*one)["createuser"]
+				email := (*user)["s_email"]
+				upload := time.Now().Format("2006-01-02 15:04:05")
+				script, _, _ = spider.GetScript(code, name, email, upload)
 			}
-		} else {
-			user, _ := u.MgoE.FindById("user", createuserid, nil)
-			name := (*one)["createuser"]
-			email := (*user)["s_email"]
-			upload := time.Now().Format("2006-01-02 15:04:05")
-			script, _, _ = spider.GetScript(id, name, email, upload)
 		}
+		f.ResponseWriter.Header().Del("Content-Type")
+		f.ResponseWriter.Header().Add("Content-Type", "application/x-download")
+		f.ResponseWriter.Header().Add("Content-Disposition", "attachment;filename=spider_"+filename)
+		f.WriteBytes([]byte(script))
+	} else {
+		f.Write("您没有权限")
+	}
+	//记录日志
+	downlogs := map[string]interface{}{
+		"code":    code,
+		"user":    user,
+		"auth":    auth,
+		"time":    time.Now().Unix(),
+		"success": success,
+		"script":  script,
 	}
-	f.ResponseWriter.Header().Del("Content-Type")
-	f.ResponseWriter.Header().Add("Content-Type", "application/x-download")
-	f.ResponseWriter.Header().Add("Content-Disposition", "attachment;filename=spider_"+filename)
-	f.WriteBytes([]byte(script))
-	// } else {
-	// 	f.Write("您没有权限")
-	// }
+	u.MgoE.Save("luadownlogs", downlogs)
 }
 
 //更新爬虫状态

src/taskManager/taskManager.go

@@ -1138,7 +1138,18 @@ func queryCriteria(userid string, urgency, taskState, event, stype int) (query b
 
 func (t *TaskM) SearchDataInfo() {
 	href := t.GetString("href")
-	data, _ := u.MgoS.FindOne("spider_warn", map[string]interface{}{"href": href})
+	stype := t.GetString("stype")
+	event, _ := t.GetInteger("event")
+	coll := "spider_warn"
+	if stype == "5" { //下载异常查列表页数据
+		if event == 7100 || event == 7110 || event == 7400 {
+			coll = "spider_highlistdata"
+		} else {
+			coll = "spider_listdata"
+		}
+	}
+	qu.Debug(coll, stype, event)
+	data, _ := u.MgoS.FindOne(coll, map[string]interface{}{"href": href})
 	if data != nil && len(*data) > 0 {
 		info := (*data)["data"].(map[string]interface{})
 		publishtime := qu.Int64All(info["publishtime"])

src/util/util.go

@@ -24,6 +24,7 @@ var (
 	TitleFilterReg2 = regexp.MustCompile(`((上|下)一页|阅读次数)`)
 	Area            []string //省份
 	DomainReg       = regexp.MustCompile(`(?://).+?(?:[:：/])`)
+	SymbolReg       = regexp.MustCompile("[,，\\s\u3000\u2003\u00a0]+")
 	ReplaceReg      = regexp.MustCompile(`[]:：/]+`)
 	CheckText       = `item["spidercode"]="%s";item["site"]="%s";item["channel"]="%s"`
 	JsonDataMap     = map[string]bool{ //jsondata

src/web/templates/head.html

@@ -637,10 +637,14 @@
   //搜索错误日志
   function searchdatainfo(){
     var href = $("#datahref").val();
+	var stype = {{.T.task.s_type}}
+	var event = {{.T.task.i_event}}
+	  console.log(stype,event)
+	  return;
     if(href==""){
       return
     }
-    $.post("/center/task/searchDataInfo",{"href":href},function(r){
+    $.post("/center/task/searchDataInfo",{"href":href,"stype":stype,"event":event},function(r){
       $("#datainfo").val(r);
       $("#datainfoarea").removeClass("hide");
     })

src/web/templates/index.html

@@ -105,6 +105,38 @@
         </div><!-- /.modal-content -->
     </div><!-- /.modal -->
 </div>
+<!-- 模态框（Modal） -->
+<div class="modal fade" id="modal-nopublish" tabindex="-1" role="dialog" aria-labelledby="myModalLabel" aria-hidden="true">
+	<div class="modal-dialog">
+		<div class="modal-content">
+			<div class="modal-header">
+				<div class="modal-header">
+					<button type="button" id="assign-close" class="close" data-dismiss="modal" aria-hidden="true">&times;</button>
+					<!--<div class="edit-form">-->
+					<div class="edit-info">
+						<span class="fa fa-scissors" id="code-assign" aria-hidden="true"></span>
+						<span class="info">爬虫无发布<span>
+					</div>
+					<form class="form-horizontal" role="form">
+						<div class="form-group">
+							<label for="modify" class="col-sm-2 control-label">爬虫:</label>
+							<div class="col-sm-10">
+								<input class="form-control" id="nopublish"></input>
+							</div>
+						</div>
+						<div class="form-group" id="assign-style">
+							<div class="col-sm-offset-2 col-sm-10">
+								<input type="button" onclick="comfirm_nopublish()" class="btn btn-primary" value="确定">
+								<input type="button" onclick="nopublish_cancel()" class="btn btn-default" value="取消">
+							</div>
+						</div>
+					</form>
+					<!--</div>	-->
+				</div>
+			</div>
+		</div><!-- /.modal-content -->
+	</div><!-- /.modal -->
+</div>
 
 <div class="content-wrapper">
 <section class="content-header">
@@ -116,6 +148,7 @@
 			<!--<a class="btn btn-default btn-sm" href="/center/spider">新建爬虫</a>-->
 			<a class="btn btn-default btn-sm" id="assign" onclick="assign()">分配</a>
 			<a class="btn btn-default btn-sm" id="disables" onclick="disables()">作废</a>
+		 <a class="btn btn-default btn-sm" onclick="nopublish()">无发布</a>
 			{{end}}
 	 </small>
    </h1>
@@ -299,11 +332,7 @@ $(function(){
 		 },
 		"columns": [
 			{ "data": "_id",render:function(val,a,row){
-				if(row.i_scope == 1){
-					return "<input type='checkbox' disabled value='"+val+"' name='"+row.param_common[1]+"_"+row.param_common[2]+"' code='"+row.param_common[0]+"'/>"
-				}else{//内部员工爬虫禁止分配
-					return "<input type='checkbox' value='"+val+"' name='"+row.param_common[1]+"_"+row.param_common[2]+"' code='"+row.param_common[0]+"'/>"
-				}
+				return "<input type='checkbox' scope='"+row.i_scope+"' value='"+val+"' name='"+row.param_common[1]+"_"+row.param_common[2]+"' code='"+row.param_common[0]+"'/>"
 			}},
       		{ "data": "_id",render:function(val,a,row){
 				return row.num
@@ -537,10 +566,15 @@ $(function(){
 		ttable.ajax.reload();
 	}
 	function selectrow(me){
-		var sel=$(me);
-		var isSelected=sel.prop('checked');
+		var sel = $(me);
+		var isSelected = sel.prop('checked');
 		if(isSelected){
 			$("#spider td input[type=checkbox]").prop("checked",true);
+			// $("#spider td input[type=checkbox]").each(function (index,item) {
+			// 	if ($(this).attr('scope') != '1') {//内部员工爬虫无法被选中
+			// 		$(this).prop("checked",true);
+			// 	}
+			// })
 		}else{
 			$("#spider td input[type=checkbox]").prop("checked",false);
 		}
@@ -548,27 +582,52 @@ $(function(){
 	function spidercopy(){
 		$("#modal-spidercopy").modal("show");
 	}
-    //复制爬虫
-	function comfirm_spidercopy(){
-    var code = $("#spidercopy").val();
-    if(code != ""){
-      $.ajax({
-				url:"/center/spider/copy",
+	function nopublish(){
+		$("#modal-nopublish").modal("show");
+	}
+	//爬虫无发布
+	function comfirm_nopublish() {
+		var code = $("#nopublish").val();
+		if(code != ""){
+			$.ajax({
+				url:"/center/spider/assort",
 				type:"post",
-				data:{"code": code},
+				data:{"code": code,"state":7},
 				success: function(r){
-				  if(r&&r.success){
-            $("#modal-spidercopy").modal("hide");
-            window.open("/center/spider/edit/"+r.encode+"__")
-          }else{
-            alert(r.text)
-          }
+					if(r&&r.success){
+						showTip("修改成功",1000);
+						$("#modal-nopublish").modal("hide");
+						ttable.ajax.reload();
+					}else{
+						alert("修改失败爬虫："+r.msg)
+					}
 				}
 			})
-    }else{
-      alert("爬虫不能为空！")
-    }
-  }
+		}else{
+			alert("爬虫不能为空！")
+		}
+	}
+    //复制爬虫
+	function comfirm_spidercopy(){
+    	var code = $("#spidercopy").val();
+		if(code != ""){
+		  $.ajax({
+			url:"/center/spider/copy",
+			type:"post",
+			data:{"code": code},
+			success: function(r){
+			  if(r&&r.success){
+				$("#modal-spidercopy").modal("hide");
+				window.open("/center/spider/edit/"+r.encode+"__")
+			  }else{
+				alert(r.text)
+			  }
+			}
+		  })
+		}else{
+		  alert("爬虫不能为空！")
+		}
+   }
   	function createComboxPendState(code){
 	return "<select onchange='changeCodePendState(this.value,\""+code+"\")' class='form-control input-sm'>"+
 			"<option value=1>是</option>"+
@@ -611,10 +670,14 @@ $(function(){
 			})
 		}
     }
+	//点击分配
 	function assign(){
 		var names=[];
 		$("#spider td input[type=checkbox]").each(function(){
-			if($(this).prop("checked")){
+			console.log($(this).attr('scope'))
+			if ($(this).attr('scope') == '1'){//禁止分配的爬虫
+				$(this).prop("checked",false);
+			}else if($(this).prop("checked")){
 				names.push($(this).attr("name"));
 			}
 		});
@@ -650,7 +713,8 @@ $(function(){
 				})
 			}*/
 		}else{
-			showTip("没有选择项", 1000, function() {});
+			showTip("没有选择项或全部爬虫无法被分配!", 2000, function() {});
+			$("#selrow").prop('checked',false);
 		}
 	}
 	//作废
@@ -693,7 +757,6 @@ $(function(){
 	function searchAssignModify(){
 		$.post("/center/searchMintainer",function(r){
 			var data = r.mintainer;
-      console.log(data)
 			$("#assign-modify").empty();
 			$("#assign-modify").append("<option value=''>--请选择--</option>");
 			for(var i in data){
@@ -710,6 +773,9 @@ $(function(){
   	function spidercopy_cancel(){
 		$("#modal-spidercopy").modal("hide");
 	}
+	function nopublish_cancel(){
+		$("#modal-nopublish").modal("hide");
+	}
 	//分配爬虫
 	function assign_save(){
 		$("#modal-assign").modal("hide");

src/web/templates/step2.html

@@ -62,7 +62,7 @@
 							<select class="form-control" name="Step2.DateFormat" id="Step2DateFormat">
 								<option value="yyyyMMddHHmmss" >年月日时分秒</option>
 								<option value="yyyyMMddHHmm" >年月日时分</option>
-								<option value="yyyyMMdd" >年月日</option>
+								<option  selected value="yyyyMMdd" >年月日</option>
 								<option value="MMdd" >月日</option>
 							</select>
 						</div>