爬虫导入功能修改；爬虫拆分功能新增

src/front/front.go

@@ -48,23 +48,24 @@ type Front struct {
 	updateESP       xweb.Mapper `xweb:"/center/spider/updateesp"`       //修改爬虫的节点/状态/平台
 	updatePendState xweb.Mapper `xweb:"/center/spider/updatePendState"` //更新爬虫挂起状态
 	tagCode         xweb.Mapper `xweb:"/center/spider/tagcode"`         //标记爬虫
+	spiderRemark    xweb.Mapper `xweb:"/center/spider/spiderremark"`    //标记爬虫
 
-	spiderModel    xweb.Mapper `xweb:"/center/model"`           //获取补充模型
-	runStep        xweb.Mapper `xweb:"/center/run"`             //方法测试
-	spiderPass     xweb.Mapper `xweb:"/center/spider/pass"`     //整体测试
-	runPinYin      xweb.Mapper `xweb:"/center/runpy"`           //获取拼音
-	saveStep       xweb.Mapper `xweb:"/center/save"`            //爬虫保存
-	saveChannels   xweb.Mapper `xweb:"/center/save/channels"`   //保存子栏目
-	saveJs         xweb.Mapper `xweb:"/center/save/js"`         //保存js
-	loadModel      xweb.Mapper `xweb:"/center/gmodel/(.*)"`     //加载模型
-	importdata     xweb.Mapper `xweb:"/center/importdata"`      //导入爬虫列表页面
-	importLua      xweb.Mapper `xweb:"/center/importlua"`       //导入爬虫
-	oldedit        xweb.Mapper `xweb:"/center/oldedit"`         //老文件编辑
-	findName       xweb.Mapper `xweb:"/center/findname"`        //即时查询名称
-	checkrepeat    xweb.Mapper `xweb:"/center/spider/isrepeat"` //脚本代码判重
-	heart          xweb.Mapper `xweb:"/center/heart"`           //心跳监控
-	spiderCopy     xweb.Mapper `xweb:"/center/spider/copy"`     //复制爬虫
-	spiderCloseErr xweb.Mapper `xweb:"/center/spider/closeerr"` //删除错误爬虫
+	spiderModel     xweb.Mapper `xweb:"/center/model"`            //获取补充模型
+	runStep         xweb.Mapper `xweb:"/center/run"`              //方法测试
+	spiderPass      xweb.Mapper `xweb:"/center/spider/pass"`      //整体测试
+	runPinYin       xweb.Mapper `xweb:"/center/runpy"`            //获取拼音
+	saveStep        xweb.Mapper `xweb:"/center/save"`             //爬虫保存
+	saveJs          xweb.Mapper `xweb:"/center/save/js"`          //保存js
+	loadModel       xweb.Mapper `xweb:"/center/gmodel/(.*)"`      //加载模型
+	importdata      xweb.Mapper `xweb:"/center/importdata"`       //导入爬虫列表页面
+	importLua       xweb.Mapper `xweb:"/center/importlua"`        //导入爬虫
+	oldedit         xweb.Mapper `xweb:"/center/oldedit"`          //老文件编辑
+	findName        xweb.Mapper `xweb:"/center/findname"`         //即时查询名称
+	checkrepeat     xweb.Mapper `xweb:"/center/spider/isrepeat"`  //脚本代码判重
+	heart           xweb.Mapper `xweb:"/center/heart"`            //心跳监控
+	spiderCopy      xweb.Mapper `xweb:"/center/spider/copy"`      //补采复制爬虫
+	spiderSplitCopy xweb.Mapper `xweb:"/center/spider/splitcopy"` //拆分复制爬虫
+	spiderCloseErr  xweb.Mapper `xweb:"/center/spider/closeerr"`  //删除错误爬虫
 
 	Base      Base
 	OtherBase OtherBase
@@ -569,9 +570,12 @@ func (f *Front) ImportLua() {
 						o["name"] = cells[0].Value
 						o["code"] = code
 						o["channel"] = cells[2].Value
-						channels := cells[3].Value
-						channels = strings.ReplaceAll(channels, "，", ",")
-						o["channels"] = strings.Split(channels, ",")
+						spiderremark := cells[3].Value
+						if spiderremark == "" {
+
+							spiderremark = `采集“` + cells[2].Value + `”栏目（含子栏目）`
+						}
+						o["spiderremark"] = spiderremark
 						//重复域名的网站不再新增爬虫
 						href := cells[4].Value
 						one, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"href": href})
@@ -703,7 +707,6 @@ func saveLua(o map[string]interface{}) (bool, string) {
 	param["site"] = o["name"]
 	param["href"] = o["channeladdr"]
 	param["channel"] = o["channel"]
-	param["channels"] = o["channels"] //子栏目
 	param["createuser"] = (*one)["s_name"]
 	param["createuserid"] = id
 	param["createuseremail"] = (*one)["s_email"]
@@ -725,6 +728,7 @@ func saveLua(o map[string]interface{}) (bool, string) {
 	param["urgency"] = o["urgency"]
 	param["isflow"] = o["isflow"]
 	param["spidertype"] = "history"
+	param["spiderremark"] = o["spiderremark"]
 	incrementevent := qu.ObjToString(o["incrementevent"])
 	if movevent, ok := util.Config.Uploadevents[incrementevent].(string); ok && movevent != "" {
 		param["spidermovevent"] = movevent
@@ -1302,3 +1306,43 @@ func (f *Front) SpiderCopy() {
 	}
 	f.ServeJson(map[string]interface{}{"success": success, "text": text, "encode": encode})
 }
+
+func (f *Front) SpiderSplitCopy() {
+	defer qu.Catch()
+	code := f.GetString("code")
+	code = strings.TrimSpace(code)
+	num, _ := f.GetInteger("num")
+	text := ""
+	lua, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
+	if len(*lua) > 0 {
+		param_common := (*lua)["param_common"].([]interface{})
+		str_list := qu.ObjToString((*lua)["str_list"])
+		channelOld := qu.ObjToString((*lua)["channel"])
+		copyNum := qu.IntAll((*lua)["copynum"]) //当前已经拆分了多少个爬虫
+		delete(*lua, "relatecode")              //删除关联爬虫字段
+		for i := 1; i <= num; i++ {
+			copyNum++
+			var codeSuffix string
+			if copyNum < 10 {
+				codeSuffix = fmt.Sprintf("0%s", fmt.Sprint(copyNum))
+
+			} else {
+				codeSuffix = fmt.Sprint(copyNum)
+			}
+			(*lua)["code"] = code + "_" + codeSuffix
+			(*lua)["channel"] = channelOld + "_" + codeSuffix
+			(*lua)["str_list"] = strings.ReplaceAll(str_list, code, code+"_"+codeSuffix)
+			(*lua)["comeintime"] = time.Now().Unix()
+			(*lua)["modifytime"] = time.Now().Unix()
+			(*lua)["state"] = 0
+			param_common[0] = code + "_" + codeSuffix
+			param_common[2] = channelOld + "_" + codeSuffix
+			(*lua)["param_common"] = param_common
+			u.MgoEB.Save("luaconfig", (*lua))
+		}
+		u.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"copynum": copyNum}}, false, false)
+	} else {
+		text = "未查询到复制原"
+	}
+	f.ServeJson(map[string]interface{}{"text": text})
+}

src/front/spider.go

@@ -173,6 +173,7 @@ func (f *Front) LoadSpider(codeTaskIdReState string) error {
 				f.T["spidertype"] = (*lua)["spidertype"]
 				f.T["spidermovevent"] = (*lua)["spidermovevent"]
 				f.T["spiderhistorymaxpage"] = (*lua)["spiderhistorymaxpage"]
+				f.T["spiderremark"] = (*lua)["spiderremark"]
 				events := []string{}
 				for k, _ := range sp.Config.Uploadevents {
 					events = append(events, k)
@@ -235,6 +236,7 @@ func (f *Front) ViewSpider(id string) error {
 				f.T["areas"] = u.Area         //
 				f.T["citys"] = u.City         //
 				f.T["provinces"] = u.Province //
+				f.T["spiderremark"] = (*lua)["spiderremark"]
 				return f.Render("spiderview.html", &f.T)
 			} else {
 				f.Write("没有对应记录！")
@@ -491,19 +493,6 @@ func (f *Front) SaveStep() {
 	}
 }
 
-func (f *Front) SaveChannels() {
-	code := f.GetString("code")
-	channels := f.GetString("channels")
-	channels = strings.ReplaceAll(channels, "，", ",")
-	arr := strings.Split(channels, ",")
-	ok := u.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{
-		"$addToSet": map[string]interface{}{
-			"channels": map[string]interface{}{"$each": arr},
-		},
-	}, false, false)
-	f.ServeJson(map[string]interface{}{"ok": ok})
-}
-
 func (f *Front) SaveJs() {
 	//param_type := f.GetString("param_type")
 	//return_type := f.GetString("return_type")

src/front/spider2.go

@@ -2,6 +2,7 @@ package front
 
 import (
 	qu "qfw/util"
+	"strings"
 	"time"
 	"util"
 )
@@ -40,3 +41,10 @@ func (f *Front) SpiderCloseErr() {
 	ok := util.MgoS.Del("spider_loadfail", nil)
 	f.ServeJson(map[string]interface{}{"ok": ok})
 }
+
+func (f *Front) SpiderRemark() {
+	defer qu.Catch()
+	remark := f.GetString("remark")
+	code := f.GetString("code")
+	util.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"spiderremark": strings.Replace(remark, "采集说明：", "", -1)}}, false, false)
+}

src/web/templates/head.html

@@ -37,6 +37,9 @@
 	<script src="/js/head.js"></script>
 	<script src="/js/xm-select.js"></script>
 	<style>
+		#spiderremark-input{
+			width: 1000px;
+		}
 		#myModal-feedback .close{
 			margin-top:-19px;
 		}
@@ -546,7 +549,7 @@
       </div>  
     </div>  
   <!-- system modal end -->  
-<div class="wrapper">
+  <div class="wrapper">
 	{{include "mask.html"}}
    <header class="main-header">
     <!-- Logo -->
@@ -561,6 +564,15 @@
       <a href="#" class="sidebar-toggle" data-toggle="offcanvas" role="button">
         <span class="sr-only">Toggle navigation</span>
       </a>
+<!--		<div>-->
+<!--			<div class="col-sm-10" style="background-color: yellow;font-weight: bold;font-size: 15px;color: #FF0000;display: inline-block;margin-top: 14px;">-->
+<!--				<label>采集说明</label>-->
+<!--				<input style="width: inherit" name="spiderremark" id="spiderremark"  value="确定">-->
+<!--			</div>-->
+<!--		</div>-->
+		<span id="spiderremark" style="background-color: yellow;font-weight: bold;font-size: 15px;color: #FF0000;display: inline-block;margin-top: 14px;">
+
+		</span>
       <!-- Navbar Right Menu -->
       <div class="navbar-custom-menu">
         <ul class="nav navbar-nav">
@@ -677,5 +689,28 @@
 		  }
 	  })
   }
+	var span = document.getElementById('spiderremark');
+	span.addEventListener('dblclick', function () {
+		var text = span.innerText;
+		var input = document.createElement('input');
+		input.id = "spiderremark-input"
+		input.value = text;
+
+		input.addEventListener('blur', function () {
+			span.innerText = input.value;
+			// input.parentNode.removeChild(input);
+			console.log("===========",input.value,)
+			var code = {{.T.lua.code}};
+			//修改内容
+			$.post("/center/spider/spiderremark",{"remark":input.value,"code":code},function (){})
+		});
+		// input.addEventListener('input', function () {
+		// 	// Handle input change
+		// 	console.log('Input changed:', this.value);
+		// });
+		span.innerText = '';
+		span.appendChild(input);
+		input.focus();
+	});
 </script>
 

src/web/templates/index.html

@@ -97,7 +97,7 @@
 					<!--<div class="edit-form">-->
 						<div class="edit-info">
 							<span class="fa fa-copy" id="code-assign" aria-hidden="true"></span>
-							<span class="info">复制爬虫<span>
+							<span class="info">补采复制<span>
 						</div>
 						<form class="form-horizontal" role="form">
 							<div class="form-group">
@@ -119,18 +119,57 @@
         </div><!-- /.modal-content -->
     </div><!-- /.modal -->
 </div>
+<!-- 拆分复制爬虫模态框（Modal） -->
+<div class="modal fade" id="modal-spidercopysplit" tabindex="-1" role="dialog" aria-labelledby="myModalLabel" aria-hidden="true">
+	<div class="modal-dialog">
+		<div class="modal-content">
+			<div class="modal-header">
+				<div class="modal-header">
+					<button type="button" class="close" data-dismiss="modal" aria-hidden="true" style="margin-top: -19px">&times;</button>
+					<!--<div class="edit-form">-->
+					<div class="edit-info">
+						<span class="fa fa-copy" id="code-assign" aria-hidden="true"></span>
+						<span class="info">拆分复制<span>
+					</div>
+					<form class="form-horizontal" role="form">
+						<div class="form-group">
+							<label class="col-sm-2 control-label">爬虫:</label>
+							<div class="col-sm-10">
+								<input class="form-control" id="spidercopysplit"></input>
+							</div>
+						</div>
+						<div class="form-group">
+							<label class="col-sm-2 control-label">数量:</label>
+							<div class="col-sm-10">
+								<input class="form-control" id="spidercopynum"></input>
+							</div>
+						</div>
+						<div class="form-group" >
+							<div class="col-sm-offset-2 col-sm-10">
+								<input type="button" onclick="comfirm_spidercopysplit()" class="btn btn-primary" value="确定">
+								<input type="button" data-dismiss="modal" class="btn btn-default" value="取消">
+							</div>
+						</div>
+					</form>
+					<!--</div>	-->
+				</div>
+			</div>
+		</div><!-- /.modal-content -->
+	</div><!-- /.modal -->
+</div>
 
 <div class="content-wrapper">
 <section class="content-header">
    <h1>
      脚本列表
      <small>
-      <a class="btn btn-default btn-sm" onclick="spidercopy()">复制爬虫</a>
-			{{if gt (session "auth") 2}}
-			<!--<a class="btn btn-default btn-sm" href="/center/spider">新建爬虫</a>-->
-			<a class="btn btn-default btn-sm" id="assign" onclick="assign()">分配爬虫</a>
-			<a class="btn btn-default btn-sm" id="disables" onclick="disables()">状态修改</a>
-			{{end}}
+      <a class="btn btn-default btn-sm" onclick="spidercopy()">补采复制</a>
+		{{if gt (session "auth") 2}}
+		<!--<a class="btn btn-default btn-sm" href="/center/spider">新建爬虫</a>-->
+		 <a class="btn btn-default btn-sm" onclick="spidercopysplit()">拆分复制</a>
+		<a class="btn btn-default btn-sm" id="assign" onclick="assign()">分配爬虫</a>
+		<a class="btn btn-default btn-sm" id="disables" onclick="disables()">状态修改</a>
+		{{end}}
 	 </small>
    </h1>
    <ol class="breadcrumb">
@@ -368,19 +407,20 @@ $(function(){
 				}
 				return  vals
 			}},
-			{ "data": "_id","width":"200px",render:function(val,a,row){
-				var text = row["channels"].join("</br>")
-				var channels = '<div align="left" style="color: #fff;">'+text+'</div>'
-				return "<a href='#' data-toggle='tooltip' data-placement='right' data-html='true'  title='"+channels+"'>"+row["channel"]+"</a>"
-			}},
-			{ "data": "_id",render:function(val,a,row){
+			// { "data": "_id","width":"200px",render:function(val,a,row){
+			// 	var text = row["channels"].join("</br>")
+			// 	var channels = '<div align="left" style="color: #fff;">'+text+'</div>'
+			// 	return "<a href='#' data-toggle='tooltip' data-placement='right' data-html='true'  title='"+channels+"'>"+row["channel"]+"</a>"
+			// }},
+			{ "data": "channel",render:function(val,a,row){
 				var param_common = row["param_common"];
 				var href = "#";
 				if (param_common[11]){
 					href = param_common[11];
 				}
-				return "<a href='"+href+"' target='_blank'>"+row["code"]+"</a>"
+				return "<a href='"+href+"' target='_blank'>"+val+"</a>"
 			}},
+			{ "data": "code"},
 		    { "data": "infoformat",render:function(val,a,row){
 				if(val == 1){
 					return "招标"
@@ -666,6 +706,9 @@ $(function(){
 	function spidercopy(){
 		$("#modal-spidercopy").modal("show");
 	}
+	function spidercopysplit() {
+		$("#modal-spidercopysplit").modal("show");
+	}
     //复制爬虫
 	function comfirm_spidercopy(){
     	var code = $("#spidercopy").val();
@@ -690,13 +733,35 @@ $(function(){
 		}else{
 		  alert("爬虫不能为空！")
 		}
-   }
+    }
+	//拆分复制爬虫
+	function comfirm_spidercopysplit(){
+		var code = $("#spidercopysplit").val();
+		var num = $("#spidercopynum").val();
+		if(code != "" || num != ""){
+			$.ajax({
+				url:"/center/spider/splitcopy",
+				type:"post",
+				data:{"code": code,"num":num},
+				success: function(r){
+					if(r&&r.text == ""){
+						// $("#modal-spidercopysplit").modal("hide");
+						window.location.reload();
+					}else{
+						alert(r.text)
+					}
+				}
+			})
+		}else{
+			alert("爬虫、数量不能为空！")
+		}
+	}
   	function createComboxPendState(code){
-	return "<select onchange='changeCodePendState(this.value,\""+code+"\")' class='form-control input-sm'>"+
+		return "<select onchange='changeCodePendState(this.value,\""+code+"\")' class='form-control input-sm'>"+
 			"<option value=1>是</option>"+
 			"<option value=0>否</option>"+
 			"</select>"
-};
+	};
 	function changeCodePendState(pendstate,code){
 		$.ajax({
 			url:"/center/spider/updatePendState",

src/web/templates/spiderbase.html

@@ -80,33 +80,7 @@
 	</div>
 	<!-- /.modal-dialog -->
 </div>
-<div class="modal fade" id="modal-channels">
-	<div class="modal-dialog">
-		<form class="form-horizontal" role="form">
-			<div class="modal-content">
-				<div class="modal-header">
-					<button type="button" class="close" data-dismiss="modal" aria-label="Close">
-						<span aria-hidden="true">&times;</span></button>
-					<h4 class="modal-title">新增</h4>
-				</div>
-				<div class="modal-body">
-					<div class="form-group">
-						<label for="channelchild" class="col-sm-2 control-label">子栏目:</label>
-						<div class="col-sm-10">
-							<input id="channelchild" name="channelchild" type="text" class="form-control" placeholder="请输入子栏目名称（栏目1,栏目2）">
-						</div>
-					</div>
-				</div>
-				<div class="modal-footer">
-					<button type="button" class="btn btn-default" data-dismiss="modal">取消</button>
-					<button type="button" class="btn btn-primary" onclick="saveChannels()">保存</button>
-				</div>
-			</div>
-			<!-- /.modal-content -->
-		</form>
-	</div>
-	<!-- /.modal-dialog -->
-</div>
+
 <div class="modal fade" id="chrome-task">
 	<div class="modal-dialog">
 		<form class="form-horizontal" role="form">
@@ -173,33 +147,6 @@
 						<input type="text" class="form-control" name="Base.SpiderChannel" id="spiderChannel"
 						placeholder="爬虫栏目" value="{{if .T.lua.param_common}}{{index .T.lua.param_common 2}}{{end}}">
 					</div>
-					<a onclick="addChannels()" style="margin-top: 10px;color: #333333;cursor:pointer;" class="fa fa-list" data-toggle="tooltip" data-placement="right" data-html="true" id="channels" title=""></a>
-					<script>
-						var text = {{.T.lua.channels}}.join("</br>")
-						var channels = '<div align="left" style="color: #fff;width: max-content">'+text+'</div>'
-						$("#channels").attr("title",channels)
-
-						function addChannels() {
-							$("#channelchild").val({{.T.lua.channels}}.join(","));
-							$("#modal-channels").modal("show");
-						}
-						function saveChannels() {
-							var channelchild = $("#channelchild").val();
-							var code = {{.T.lua.code}};
-							$.post("/center/save/channels",{"channels":channelchild,"code":code},function(r){
-								if (r&&r.ok){
-									$('#modal-channels').modal("hide");
-									showTip("保存成功", 1000);
-									setTimeout(function (){
-										window.location.reload();
-									}, 1000);
-								}else{
-									showTip("保存失败", 1000);
-								}
-							})
-						}
-						
-					</script>
 				</div>
         		<div class="form-group">
 					<label for="isflow" class="col-sm-6 control-label text-right">

src/web/templates/spideredit.html

@@ -265,6 +265,10 @@
 	var areas = {{.T.provinces}};
 	var citys = {{.T.citys}};
 	$(function(){
+		//采集说明
+		if ({{.T.spiderremark}} && {{.T.spiderremark}} != ""){
+			$("#spiderremark").text("采集说明："+{{.T.spiderremark}});
+		}
 		editor2_js = CodeMirror.fromTextArea($("#Step2_js")[0], {
 			theme:"solarized light",
 			lineNumbers: true,

src/web/templates/spiderview.html

@@ -69,6 +69,10 @@
 	var areas = {{.T.provinces}};
 	var citys = {{.T.citys}};
 	$(function(){
+		//采集说明
+		if ({{.T.spiderremark}} && {{.T.spiderremark}} != ""){
+			$("#spiderremark").text("采集说明："+{{.T.spiderremark}});
+		}
 		editor2_js = CodeMirror.fromTextArea($("#Step2_js")[0], {
 			theme:"solarized light",
 			lineNumbers: true,