站点列表更新

src/front/front.go

@@ -47,19 +47,20 @@ type Front struct {
 	updatePendState xweb.Mapper `xweb:"/center/spider/updatePendState"` //更新爬虫挂起状态
 	tagCode         xweb.Mapper `xweb:"/center/spider/tagcode"`         //标记爬虫
 
-	spiderModel xweb.Mapper `xweb:"/center/model"`           //获取补充模型
-	runStep     xweb.Mapper `xweb:"/center/run"`             //方法测试
-	spiderPass  xweb.Mapper `xweb:"/center/spider/pass"`     //整体测试
-	runPinYin   xweb.Mapper `xweb:"/center/runpy"`           //获取拼音
-	saveStep    xweb.Mapper `xweb:"/center/save"`            //爬虫保存
-	loadModel   xweb.Mapper `xweb:"/center/gmodel/(.*)"`     //加载模型
-	importdata  xweb.Mapper `xweb:"/center/importdata"`      //导入脚本
-	importfile  xweb.Mapper `xweb:"/center/importfile"`      //批量导入爬虫
-	oldedit     xweb.Mapper `xweb:"/center/oldedit"`         //老文件编辑
-	findName    xweb.Mapper `xweb:"/center/findname"`        //即时查询名称
-	checkrepeat xweb.Mapper `xweb:"/center/spider/isrepeat"` //脚本代码判重
-	heart       xweb.Mapper `xweb:"/center/heart"`           //心跳监控
-	spiderCopy  xweb.Mapper `xweb:"/center/spider/copy"`     //复制爬虫
+	spiderModel  xweb.Mapper `xweb:"/center/model"`           //获取补充模型
+	runStep      xweb.Mapper `xweb:"/center/run"`             //方法测试
+	spiderPass   xweb.Mapper `xweb:"/center/spider/pass"`     //整体测试
+	runPinYin    xweb.Mapper `xweb:"/center/runpy"`           //获取拼音
+	saveStep     xweb.Mapper `xweb:"/center/save"`            //爬虫保存
+	saveChannels xweb.Mapper `xweb:"/center/save/channels"`   //保存子栏目
+	loadModel    xweb.Mapper `xweb:"/center/gmodel/(.*)"`     //加载模型
+	importdata   xweb.Mapper `xweb:"/center/importdata"`      //导入脚本
+	importfile   xweb.Mapper `xweb:"/center/importfile"`      //批量导入爬虫
+	oldedit      xweb.Mapper `xweb:"/center/oldedit"`         //老文件编辑
+	findName     xweb.Mapper `xweb:"/center/findname"`        //即时查询名称
+	checkrepeat  xweb.Mapper `xweb:"/center/spider/isrepeat"` //脚本代码判重
+	heart        xweb.Mapper `xweb:"/center/heart"`           //心跳监控
+	spiderCopy   xweb.Mapper `xweb:"/center/spider/copy"`     //复制爬虫
 
 	Base      Base
 	OtherBase OtherBase
@@ -430,7 +431,7 @@ func (f *Front) LoadIndex() {
 		qu.Debug("query:", query, "sort:", sort)
 		//luas := *mgdb.Find("luaconfig", query, sort, list_fields, false, start, limit)
 		//count := mgdb.Count("luaconfig", query)
-		luas, _ := u.MgoEB.Find("luaconfig", query, sort, list_fields, false, start, limit)
+		luas, _ := u.MgoEB.Find("luaconfig", query, sort, map[string]interface{}{"str_list": 0, "str_content": 0}, false, start, limit)
 		count := u.MgoEB.Count("luaconfig", query)
 		for k, v := range *luas {
 			v["num"] = k + 1 + page*10
@@ -541,37 +542,40 @@ func (f *Front) Importfile() {
 						o["name"] = cells[0].Value
 						o["code"] = cells[1].Value
 						o["channel"] = cells[2].Value
+						channels := cells[3].Value
+						channels = strings.ReplaceAll(channels, "，", ",")
+						o["channels"] = strings.Split(channels, ",")
 						//重复域名的网站不再新增爬虫
-						href := cells[3].Value
+						href := cells[4].Value
 						one, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"href": href})
 						if len(*one) > 0 {
 							errorinfo[cells[1].Value] = "第" + strconv.Itoa(k) + "行爬虫" + cells[1].Value + ",该栏目已存在,请确认"
 						}
 						o["channeladdr"] = href
-						o["author"] = cells[4].Value
+						o["author"] = cells[5].Value
 						o["timestamp"] = time.Now().Unix()
 						o["status"] = 1
-						o["next"] = cells[4].Value
-						o["event"] = cells[5].Value
-						o["incrementevent"] = cells[6].Value
-						if cells[7].Value == "是" {
+						o["next"] = cells[5].Value
+						o["event"] = cells[6].Value
+						o["incrementevent"] = cells[7].Value
+						if cells[8].Value == "是" {
 							o["isflow"] = 1
 						} else {
 							o["isflow"] = 0
 						}
-						if cells[8].Value == "紧急" {
+						if cells[9].Value == "紧急" {
 							o["urgency"] = 1
 						} else {
 							o["urgency"] = 0
 						}
-						o["platform"] = cells[9].Value
-						o["area"] = cells[10].Value
-						o["city"] = cells[11].Value
-						o["district"] = cells[12].Value
-						weigh, _ := cells[13].Int()
+						o["platform"] = cells[10].Value
+						o["area"] = cells[11].Value
+						o["city"] = cells[12].Value
+						o["district"] = cells[13].Value
+						weigh, _ := cells[14].Int()
 						o["weight"] = weigh
 						//存储表
-						o["coll"] = cells[14].Value
+						o["coll"] = cells[15].Value
 						//table := cells[6].Value
 						//o["table"] = table
 						//o["transfercode"] = qu.IntAll(Transfercode[table])
@@ -665,6 +669,7 @@ func savelua(o map[string]interface{}) (bool, string) {
 	param["site"] = o["name"]
 	param["href"] = o["channeladdr"]
 	param["channel"] = o["channel"]
+	param["channels"] = o["channels"] //子栏目
 	param["createuser"] = (*one)["s_name"]
 	param["createuserid"] = id
 	param["createuseremail"] = (*one)["s_email"]

src/front/site.go

@@ -40,7 +40,7 @@ func (s *Site) SiteList() {
 		if event != "-1" && event != "" {
 			query["event"] = map[string]interface{}{"$regex": event}
 		}
-		if sitestype != "-1" && sitestype != "" {
+		if sitestype != "-1" {
 			query["site_type"] = sitestype
 		}
 		if area != "-1" && area != "" {
@@ -108,6 +108,7 @@ func (s *Site) SaveSite() {
 	othername := s.GetString("othername")
 	domain := s.GetString("domain")
 	area := s.GetString("area")
+	qy_area := s.GetString("qy_area")
 	city := s.GetString("city")
 	district := s.GetString("district")
 	industry := s.GetString("industry")
@@ -128,6 +129,7 @@ func (s *Site) SaveSite() {
 		"another_name": othername,
 		"domain":       domain,
 		"area":         area,
+		"qy_area":      qy_area,
 		"city":         city,
 		"district":     district,
 		"industry":     industry,

src/front/spider.go

@@ -437,6 +437,19 @@ func (f *Front) SaveStep() {
 	}
 }
 
+func (f *Front) SaveChannels() {
+	code := f.GetString("code")
+	channels := f.GetString("channels")
+	channels = strings.ReplaceAll(channels, "，", ",")
+	arr := strings.Split(channels, ",")
+	ok := u.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{
+		"$addToSet": map[string]interface{}{
+			"channels": map[string]interface{}{"$each": arr},
+		},
+	}, false, false)
+	f.ServeJson(map[string]interface{}{"ok": ok})
+}
+
 func LuaSaveLog(code, user string, data *map[string]interface{}, stype int) {
 	saveOne, _ := u.MgoEB.FindOne("luasavelog", map[string]interface{}{"state": 0, "code": code})
 	if stype == 0 { //保存记录

src/spider/script.go

@@ -15,6 +15,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/shopspring/decimal"
+	"io"
 	"io/ioutil"
 	mu "mfw/util"
 	"mime/multipart"
@@ -324,6 +325,28 @@ func (s *Script) LoadScript(downloadnode, script string, isfile ...string) {
 		S.Push(lua.LString(fid))
 		return 5
 	}))
+	s.L.SetGlobal("downloadBase64File", s.L.NewFunction(func(S *lua.LState) int {
+		url := S.ToString(-3)
+		fileName := S.ToString(-2)
+		base64Img := S.ToString(-1)
+		if fileName == "" {
+			fileName = "文件下载"
+		}
+		fileName = fileName + ".jpg"
+		i := strings.Index(base64Img, ",")
+		dec := base64.NewDecoder(base64.StdEncoding, strings.NewReader(base64Img[i+1:]))
+		ret, err := io.ReadAll(dec)
+		name, size, ftype, fid := "", "", "", ""
+		if err == nil && len(ret) > 0 {
+			url, name, size, ftype, fid = util.UploadFile(s.SCode, fileName, url, ret)
+		}
+		S.Push(lua.LString(url))
+		S.Push(lua.LString(name))
+		S.Push(lua.LString(size))
+		S.Push(lua.LString(ftype))
+		S.Push(lua.LString(fid))
+		return 5
+	}))
 	//支持正则
 	s.L.SetGlobal("regexp", s.L.NewFunction(func(S *lua.LState) int {
 		index := int(S.ToNumber(-1))

src/util/util.go

@@ -111,6 +111,7 @@ func InitMgo() {
 //初始化省市行政区划信息
 func InitAreaCity() {
 	qu.ReadConfig("areacity.json", &Province)
+	Area = append(Area, "全国")
 	for area, _ := range Province {
 		if area == "全国" {
 			continue

src/web/templates/index.html

@@ -355,19 +355,19 @@ $(function(){
 				}
 				return  vals
 			}},
-			{ "data": "param_common","width":"200px",render:function(val){
-				if(val.length<3){
-					val="暂无";
-				}else{
-					var href="#"
-					if(val[11]){
-						href=val[11]
-					}
-					val="<a href='"+href+"' title='"+val[2]+"' target='_blank'>"+val[2]+"</a>"
+			{ "data": "_id","width":"200px",render:function(val,a,row){
+				var text = row["channels"].join("</br>")
+				var channels = '<div align="left" style="color: #fff;">'+text+'</div>'
+				return "<a href='#' data-toggle='tooltip' data-placement='right' data-html='true'  title='"+channels+"'>"+row["channel"]+"</a>"
+			}},
+			{ "data": "_id",render:function(val,a,row){
+				var param_common = row["param_common"];
+				var href = "#";
+				if (param_common[11]){
+					href = param_common[11];
 				}
-				return val
+				return "<a href='"+href+"' target='_blank'>"+row["code"]+"</a>"
 			}},
-			{ "data": "code"},
 			{ "data": "urgency",render:function(val){
 				if(val==1){
 				  return "紧急"
@@ -472,7 +472,7 @@ $(function(){
 				{{end}}
 				return div.html()
 			}}
-    ],
+    	],
 		"fnDrawCallback": function(table) {
 		 	$("ul.pagination").prepend("&nbsp;&nbsp;&nbsp;转到第 <input type='text' id='changePage'   style='width:20px;'> 页    <a type='text' href='javascript:void(0);' id='dataTable-btn' style='text-align:center'>GO</a>");
 		 	$('#dataTable-btn').click(function(e) {    

src/web/templates/sitelist.html

@@ -35,6 +35,12 @@
                                     <input type="text" class="form-control" id="edit-area" placeholder="请输入省份">
                                 </div>
                             </div>
+                            <div class="form-group">
+                                <label for="qy_area" class="col-sm-2 control-label">辅助省份:</label>
+                                <div class="col-sm-10">
+                                    <input type="text" class="form-control" id="edit-qy_area" placeholder="请输入辅助省份">
+                                </div>
+                            </div>
                             <div class="form-group">
                                 <label for="city" class="col-sm-2 control-label">城市:</label>
                                 <div class="col-sm-10">
@@ -120,6 +126,12 @@
                                     <input type="text" class="form-control" id="area" placeholder="请输入省份">
                                 </div>
                             </div>
+                            <div class="form-group">
+                                <label for="qy_area" class="col-sm-2 control-label">辅助省份:</label>
+                                <div class="col-sm-10">
+                                    <input type="text" class="form-control" id="qy_area" placeholder="请输入辅助省份">
+                                </div>
+                            </div>
                             <div class="form-group">
                                 <label for="city" class="col-sm-2 control-label">城市:</label>
                                 <div class="col-sm-10">
@@ -204,6 +216,7 @@
                     <th>别名</th>
                     <th>域名</th>
                     <th>省份</th>
+                    <th>辅助省份</th>
                     <th>城市</th>
                     <th>区县</th>
                     <th>行业</th>
@@ -237,7 +250,7 @@ $(function(){
               "url": "/js/dataTables.chinese.lang"
           },
 		"columnDefs": [
-		     { "orderable": false, "targets": [0,1,3,4,5,6,7,8,9,10,11,13,14] },
+		     { "orderable": false, "targets": [0,1,3,4,5,6,7,8,9,10,11,12,14,15] },
             //爬虫挂起
             {"targets":[3], createdCell: function (cell, cellData, rowData, rowIndex, colIndex) {
                 var aInput;
@@ -264,7 +277,7 @@ $(function(){
                 });
             }}
 		],
-		"order": [[12,"desc"]],
+		"order": [[13,"desc"]],
         "lengthChange":false,
 		"processing":true,
 		"serverSide": true,
@@ -296,6 +309,13 @@ $(function(){
           return "<a href='http://"+val+"' target='_blank' style='color: #333'>"+val+"</a>"
       }},
       { "data": "area"},
+      { "data": "qy_area",render:function (val,a,row) {
+          if(val){
+              return val;
+          }else{
+              return "";
+          }
+          }},
       { "data": "city"},
       { "data": "district"},
       { "data": "industry"},
@@ -351,11 +371,16 @@ $(function(){
             }else{
                 e.event="-1"
             }
+            console.log("sitestype",sitestype)
             if(sitestype){
+                if (sitestype == "空"){
+                    sitestype = "";
+                }
                 e.sitestype = sitestype;
             }else{
                 e.sitestype="-1";
             }
+
             if(area){
                 e.area = area;
             }else{
@@ -393,6 +418,7 @@ $(function(){
         for(k in sitetypes){
             sitestype+="<option value='"+sitetypes[k]+"'>"+sitetypes[k]+"</option>";
         }
+        sitestype+="<option value='空'>空</option>";
         var selectStype="<div class='form-group'><label for='name'>网站类型：</label>"+
             "<select id='sitestype' onchange='checkclick(this.value)' class='form-control input-sm'>"+
             sitestype+
@@ -430,6 +456,7 @@ function updateSite(){
     var domain = $("#edit-domain").val();
     var othername = $("#edit-othername").val();
     var area = $("#edit-area").val();
+    var qy_area = $("#edit-qy_area").val();
     var city = $("#edit-city").val();
     var district = $("#edit-district").val();
     var industry = $("#edit-industry").val();
@@ -439,7 +466,7 @@ function updateSite(){
     $.ajax({
         url:"/center/site/savesite",
         type:"post",
-        data:{"id":id,"site":site,"sitetype":sitetype,"othername":othername,"domain":domain,"area":area,"city":city,"district":district,"industry":industry,"psite":psite,"ssite":ssite},
+        data:{"id":id,"site":site,"sitetype":sitetype,"othername":othername,"domain":domain,"area":area,"qy_area":qy_area,"city":city,"district":district,"industry":industry,"psite":psite,"ssite":ssite},
         success:function (r){
             if(r&&r.ok){
                 showTip("保存成功", 1000);
@@ -460,6 +487,7 @@ function siteEdit(rowjson){
     $("#edit-domain").val(row.domain);
     $("#edit-othername").val(row.another_name);
     $("#edit-area").val(row.area);
+    $("#edit-qy_area").val(row.qy_area);
     $("#edit-city").val(row.city);
     $("#edit-district").val(row.district);
     $("#edit-industry").val(row.industry);
@@ -518,6 +546,7 @@ function saveSite(id){
     var domain = $("#domain").val();
     var othername = $("#othername").val();
     var area = $("#area").val();
+    var qy_area = $("#qy_area").val();
     var city = $("#city").val();
     var district = $("#district").val();
     var industry = $("#industry").val();
@@ -527,7 +556,7 @@ function saveSite(id){
     $.ajax({
         url:"/center/site/savesite",
         type:"post",
-        data:{"id":id,"site":site,"sitetype":sitetype,"othername":othername,"domain":domain,"area":area,"city":city,"district":district,"industry":industry,"psite":psite,"ssite":ssite},
+        data:{"id":id,"site":site,"sitetype":sitetype,"othername":othername,"domain":domain,"area":area,"qy_area":qy_area,"city":city,"district":district,"industry":industry,"psite":psite,"ssite":ssite},
         success:function (r){
             if(r&&r.ok){
                 showTip("保存成功", 1000000);

src/web/templates/spiderbase.html

@@ -1,3 +1,31 @@
+<div class="modal fade" id="modal-channels">
+	<div class="modal-dialog">
+		<form class="form-horizontal" role="form">
+			<div class="modal-content">
+				<div class="modal-header">
+					<button type="button" class="close" data-dismiss="modal" aria-label="Close">
+						<span aria-hidden="true">&times;</span></button>
+					<h4 class="modal-title">新增</h4>
+				</div>
+				<div class="modal-body">
+					<div class="form-group">
+						<label for="channelchild" class="col-sm-2 control-label">子栏目:</label>
+						<div class="col-sm-10">
+							<input id="channelchild" name="channelchild" type="text" class="form-control" placeholder="请输入子栏目名称（栏目1,栏目2）">
+						</div>
+					</div>
+				</div>
+				<div class="modal-footer">
+					<button type="button" class="btn btn-default" data-dismiss="modal">取消</button>
+					<button type="button" class="btn btn-primary" onclick="saveChannels()">保存</button>
+				</div>
+			</div>
+			<!-- /.modal-content -->
+		</form>
+	</div>
+	<!-- /.modal-dialog -->
+</div>
+<!-- /.modal -->
 <form class="form-horizontal" id="baseinfo">
 	<div class="col-md-4 max-base">
 		<div class="box box-primary">
@@ -35,6 +63,33 @@
 						<input type="text" class="form-control" name="Base.SpiderChannel" id="spiderChannel"
 						placeholder="爬虫栏目" value="{{if .T.lua.param_common}}{{index .T.lua.param_common 2}}{{end}}">
 					</div>
+					<a onclick="addChannels()" style="margin-top: 10px;color: #333333;cursor:pointer;" class="fa fa-list" data-toggle="tooltip" data-placement="right" data-html="true" id="channels" title=""></a>
+					<script>
+						var text = {{.T.lua.channels}}.join("</br>")
+						var channels = '<div align="left" style="color: #fff;width: max-content">'+text+'</div>'
+						$("#channels").attr("title",channels)
+
+						function addChannels() {
+							$("#channelchild").val({{.T.lua.channels}}.join(","));
+							$("#modal-channels").modal("show");
+						}
+						function saveChannels() {
+							var channelchild = $("#channelchild").val();
+							var code = {{.T.lua.code}};
+							$.post("/center/save/channels",{"channels":channelchild,"code":code},function(r){
+								if (r&&r.ok){
+									$('#modal-channels').modal("hide");
+									showTip("保存成功", 1000);
+									setTimeout(function (){
+										window.location.reload();
+									}, 1000);
+								}else{
+									showTip("保存失败", 1000);
+								}
+							})
+						}
+						
+					</script>
 				</div>
         <div class="form-group">
 					<label for="isflow" class="col-sm-6 control-label text-right">
@@ -385,6 +440,7 @@
 	</div>
 </form>
 <script>
+
   function spiderTypeChange(){
     var val = $("#spiderType").val();
     if (val == "increment") {//选择增量模式