支持数据检索

src/config.json

@@ -62,6 +62,11 @@
     "dbsize": 5,
     "dbname": "spider"
   },
+  "python_db": {
+    "addr": "192.168.3.207:27092",
+    "dbsize": 5,
+    "dbname": "py_spider"
+  },
   "indexNode": {
     "addr": "127.0.0.1",
     "port": 1483,

src/lua/data.go

@@ -0,0 +1,77 @@
+package lua
+
+import (
+	qu "qfw/util"
+	"strings"
+	"util"
+)
+
+func (l *Lua) DataView() {
+	defer qu.Catch()
+	coll := l.GetString("coll")
+	id := l.GetString("id")
+	data := &map[string]interface{}{}
+	if coll == "bidding" {
+		data, _ = util.JYMgo.FindById(coll, id, nil)
+	} else {
+		if tmp := strings.Split(coll, "-"); len(tmp) == 2 {
+			if tmp[0] == "lua" {
+				data, _ = util.MgoS.FindById(tmp[1], id, nil)
+			} else if tmp[0] == "python" {
+				data, _ = util.MgoPy.FindById(tmp[1], id, nil)
+			}
+		} else {
+			data, _ = util.MgoS.FindById(coll, id, nil)
+		}
+	}
+	l.T["data"] = *data
+	l.Render("lua/dataview.html", &l.T)
+}
+
+func (l *Lua) DataFind() {
+	defer qu.Catch()
+	l.Render("lua/datafind.html")
+}
+func (l *Lua) SearchData() {
+	defer qu.Catch()
+	text := l.GetString("text")
+	stype := l.GetString("stype")
+	data := []map[string]interface{}{}
+	query := map[string]interface{}{
+		stype: text,
+	}
+	if stype == "title" { //根据title找bidding(bidding表有title索引，无href索引)
+		query = map[string]interface{}{
+			"title": text,
+		}
+		//1、根据title找bidding
+		list_bid, _ := util.JYMgo.Find("bidding", query, nil, nil, false, -1, -1)
+		AddVal(*list_bid, "bidding", &data)
+	}
+	//2、找spider_repeatdata判重数据表
+	list_repeat, _ := util.MgoS.Find("spider_repeatdata", query, nil, nil, false, -1, -1)
+	AddVal(*list_repeat, "spider_repeatdata", &data)
+	//3、找spider_warn异常数据表
+	list_warn, _ := util.MgoS.Find("spider_warn", query, nil, nil, false, -1, -1)
+	AddVal(*list_warn, "spider_warn", &data)
+	//4、找data_bak表
+	list_bak_lua, _ := util.MgoS.Find("data_bak", query, nil, nil, false, -1, -1)
+	AddVal(*list_bak_lua, "lua-data_bak", &data)
+	list_bak_py, _ := util.MgoPy.Find("data_bak", query, nil, nil, false, -1, -1)
+	AddVal(*list_bak_py, "python-data_bak", &data)
+	//if len(data) == 0 { //以上都没有找到数据，找列表页数据
+	//
+	//}
+	l.ServeJson(map[string]interface{}{
+		"data": data,
+	})
+}
+
+func AddVal(list []map[string]interface{}, by string, data *[]map[string]interface{}) {
+	if listLen := len(list); listLen > 0 {
+		for _, l := range list {
+			l["by"] = by
+		}
+		*data = append(*data, list...)
+	}
+}

src/lua/spiderwarn.go

@@ -26,6 +26,12 @@ type Lua struct {
 	spiderEdit   xweb.Mapper `xweb:"/lua/spideredit"`   //编辑异常数据
 	spiderSave   xweb.Mapper `xweb:"/lua/spiersave"`    //数据更新、保存
 	spiderUpdate xweb.Mapper `xweb:"/lua/spiderupdate"` //批量更新状态
+
+	//data
+	dataFind   xweb.Mapper `xweb:"/lua/datafind"`   //数据查找页面
+	searchData xweb.Mapper `xweb:"/lua/searchdata"` //通过title、href查找
+	dataView   xweb.Mapper `xweb:"/lua/dataview"`   //查看数据详情
+
 }
 
 func (l *Lua) SpiderWarn() {

src/util/config.go

@@ -17,6 +17,7 @@ var (
 	MgoEn           *mongodb.MongodbSim
 	JyProMgo        *mongodb.MongodbSim
 	MgoS            *mongodb.MongodbSim
+	MgoPy           *mongodb.MongodbSim
 	EsIndex, EsType string
 	MgoEnC          string
 	JyCollNameOne   string
@@ -70,6 +71,14 @@ func InitMgoPool() {
 		DbName:      qu.ObjToString(luaDb["dbname"]),
 	}
 	MgoS.InitPool()
+	//python
+	pythonDb := *qu.ObjToMap(Sysconfig["python_db"])
+	MgoPy = &mongodb.MongodbSim{
+		MongodbAddr: qu.ObjToString(pythonDb["addr"]),
+		Size:        qu.IntAll(pythonDb["dbsize"]),
+		DbName:      qu.ObjToString(pythonDb["dbname"]),
+	}
+	MgoPy.InitPool()
 }
 
 func initJYMgo() {

src/web/templates/lua/datafind.html

@@ -0,0 +1,236 @@
+{{include "com/inc.html"}}
+<!-- Main Header -->
+{{include "com/header.html"}}
+<!-- Left side column. 权限菜单 -->
+{{include "com/menu.html"}}
+
+<style>
+    .j-input__inner {
+        box-sizing: border-box;
+        display: inline-block;
+        padding: 0px 45px 0px 15px;
+        width: 300px;
+        height: 40px;
+        color: #1D1D1D;
+
+        background-color: #fff;
+        background-image: none;
+        line-height: 40px;
+        font-size: inherit;
+        -webkit-appearance: none;
+        outline: none;
+        border: 1px solid #dcdfe6;
+        border-radius: 4px;
+        transition: border-color 0.2s cubic-bezier(0.645, 0.045, 0.355, 1);
+    }
+    .j-input__inner:hover {
+        border-color: #c0c4cc;
+    }
+    .j-input__inner:focus {
+        border-color: #409eff;
+    }
+    .search-icon {
+        position:relative;
+        width: 30px;
+        height: 30px;
+        top: 0px;
+        right: 40px;
+    }
+</style>
+
+<div class="content-wrapper" id="showbtn">
+    <section class="content-header">
+        <ol class="breadcrumb">
+            <li><a href="#"><i class="fa fa-dashboard"></i> 首页</a></li>
+            <li><a href="/lua/datafind"> 数据查询</a></li>
+        </ol>
+
+        <br/>
+    </section>
+    <!-- Main content -->
+    <section class="content">
+        <div class="row">
+            <div class="col-xs-12">
+                <div class="box">
+                    <div class="box-body">
+                        <div class="tab-content">
+                            <input class="j-input__inner" type="text" placeholder="请输入标题" id="title">
+                            <img src="/picture/img_search.jpg" style='cursor:pointer;' class="search-icon" onclick="searchData('title')" />
+                            <input class="j-input__inner" type="text" placeholder="请输入链接" id="href">
+                            <img src="/picture/img_search.jpg" style='cursor:pointer;' class="search-icon" onclick="searchData('href')" />
+                        </div>
+
+                        <table id="dataTable" class="table table-bordered table-hover">
+                            <thead>
+                            <tr>
+                                <th>编号</th>
+                                <th>id</th>
+                                <th>标题</th>
+                                <th>链接</th>
+                                <th>来源</th>
+                                <th>操作</th>
+                            </tr>
+                            </thead>
+                        </table>
+                    </div>
+                    <!-- /.box-body -->
+                </div>
+                <!-- /.box -->
+            </div>
+        </div>
+    </section>
+</div>
+
+{{include "com/footer.html"}}
+<script>
+    menuActive("datafind");
+    $(document).ready(function () {
+        ttable = $('#dataTable').DataTable({
+            "paging": false,
+            "lengthChange": false,
+            "searching": false,
+            "ordering": false,
+            "info": true,
+            "autoWidth": false,
+            "serverSide": false,
+            "language": {
+                "url": "/dist/js/dataTables.chinese.lang"
+            },
+            "fnDrawCallback": function () {
+                $("ul.pagination").prepend("&nbsp;&nbsp;&nbsp;转到第 <input type='text' id='changePage'   style='width:20px;'> 页    <a type='text' href='javascript:void(0);' id='dataTable-btn' style='text-align:center'>GO</a>");
+                $('#dataTable-btn').click(function (e) {
+                    var redirectpage = 0
+                    if ($("#changePage").val() && $("#changePage").val() > 0) {
+                        var redirectpage = $("#changePage").val() - 1;
+                    }
+                    ttable.page(redirectpage).draw(false);
+                });
+                this.api().column(0).nodes().each(function(cell, i) {
+                    cell.innerHTML = i + 1;
+                });
+            },
+            "columns": [
+                {"data": null,width:"5%"},
+                {"data": "_id"},
+                {"data": "title", width: "50%"},
+                {"data": function (row) {
+                        if (row.href) {
+                            var str = "链接地址"
+                            tmp = '<a class="" target="_blank" href=' + row.href + '>' + str + '</a>';
+                            return tmp
+                        } else {
+                            return ""
+                        }
+                    },  width: "10%"},
+                {"data": "by"},
+                {"data": function (row) {
+                        return "<a href='/lua/dataview?coll="+row.by+"&id="+row._id+"' target='_blank' class='btn btn-sm btn-primary'>详情</a>"
+                    }},
+            ]
+        });
+    });
+
+    function searchData(stype) {
+        var text = ""
+        if(stype == "title"){
+            text = $('#title').val()
+        }else if(stype == "href"){
+            text = $('#href').val()
+        }
+        if(text == ""){
+            alert("内容不能为空")
+            return
+        }
+
+        $.ajax({
+            url:"/lua/searchdata",
+            type:"post",
+            data:{"text": text,"stype":stype},
+            success:function(r){
+                if (r&&r.data.length>0) {
+                    $('#dataTable').dataTable().fnClearTable();
+                    $('#dataTable').dataTable().fnAddData(r.data);
+                    $('#dataTable').dataTable().fn
+                }else {
+                    showTip("未找到数据")
+                }
+            }
+        })
+    }
+    function idSearch() {
+        var search_name = $('#idSea').val()
+        $.ajax({
+            url:"/service/jy/searchID",
+            type:"post",
+            data:{"_id": search_name},
+            success:function(r){
+                if (r.rep) {
+                    curColl = r.orgColl
+                    $('#dataTable').dataTable().fnClearTable();
+                    $('#dataTable').dataTable().fnAddData(r.data);
+                }else {
+                    showTip(r.msg)
+                }
+            }
+        })
+    }
+    function jyurlSearch() {
+        var search_name = $('#jyurlSea').val()
+        $.ajax({
+            url:"/service/jy/searchJyurl",
+            type:"post",
+            data:{"jyurl": search_name},
+            success:function(r){
+                if (r.rep) {
+                    curColl = r.orgColl
+                    $('#dataTable').dataTable().fnClearTable();
+                    $('#dataTable').dataTable().fnAddData(r.data);
+                }else {
+                    showTip(r.msg)
+                }
+            }
+        })
+    }
+
+    function jyShaSearch() {
+        var search_name = $('#shaSea').val()
+        $.ajax({
+            url:"/service/jy/searchSha",
+            type:"post",
+            data:{"sha": search_name},
+            success:function(r){
+                if (r.rep) {
+                    curColl = "bidding"
+                    $('#dataTable').dataTable().fnClearTable();
+                    $('#dataTable').dataTable().fnAddData(r.data);
+                }else {
+                    showTip(r.msg)
+                }
+            }
+        })
+    }
+
+    function del_repair(del_id,coll){
+        showConfirm("确定删除？", function() {
+            $.ajax({
+                url:"/service/jy/delete",
+                type:"post",
+                data:{"_id": del_id,"coll":curColl},
+                success:function(r){
+                    if(r.rep){
+                        $('#dataTable').dataTable().fnClearTable();
+                        showTip("删除成功")
+                    }else{
+                        alert("删除失败");
+                    }
+                }
+            })
+        });
+    }
+
+
+
+
+
+
+</script>

src/web/templates/lua/dataview.html

@@ -0,0 +1,187 @@
+{{include "com/inc.html"}}
+<!-- Main Header -->
+{{include "com/header.html"}}
+<!-- Left side column. 权限菜单 -->
+{{include "com/menu.html"}}
+
+<style>
+    /* 方法1：设置textarea合适的宽高 */
+    #jsonTextarea {
+        float: left;
+        margin-right: 20px;
+        width: 40%;
+        height: 70vh;
+        outline: none;
+        padding: 5px;
+    }
+
+    /* 方法2：自定义高亮样式 */
+    #jsonPre {
+        float: left;
+        width: 40%;
+        height: 70vh;
+        outline: 1px solid #ccc;
+        padding: 5px;
+        overflow: scroll;
+    }
+</style>
+
+
+<div class="content-wrapper">
+    <section class="content-header">
+        <h1>
+            数据详情
+        </h1>
+        <ol class="breadcrumb">
+            <li><a href="#"><i class="fa fa-dashboard"></i> 首页</a></li>
+            <li><a href="/lua/datafind"> 数据查询</a></li>
+            <li><a href="#"> 数据详情</a></li>
+        </ol>
+    </section>
+    <!-- Main content -->
+    <section class="content">
+        <div class="nav-tabs-custom">
+            <div class="box-body">
+
+            </div>
+            <form class="form-horizontal">
+                <div class="box-body">
+                    <textarea id="jsonTextarea" style="width: 100%;"></textarea>
+                </div>
+            </form>
+        </div>
+    </section>
+</div>
+
+{{include "com/footer.html"}}
+<script>
+    menuActive("datafind");
+    var edit_data = {{.T.data}}
+    var edit_contentHtml = {{.T.contenthtml}}
+    var edit_detail = {{.T.detail}}
+    var edit_coll = {{.T.coll}}
+    var edit_summary = {{.T.summary}}
+    var sid = {{.T.id}}
+    function parse(str) {
+        return JSON.stringify(str, null, "\t")
+    }
+    $('#jsonTextarea').val(parse(edit_data)); //其他属性
+    $('#jsonDetail').val(edit_detail); //其他属性
+    $('#jsonContentHtml').val(edit_contentHtml); //其他属性
+    $('#summary').val(edit_summary);
+
+    //更新
+    function saveRepair() {
+
+        var reason=$("#reasons").val()
+        if (reason=="") {
+            let msg = "请填写新增数据原因"
+            showMsg(msg)
+            return;
+        }
+
+        var curData = document.getElementById('jsonTextarea').value;
+        var data = JSON.parse(curData)
+        // 处理特殊字段
+        if (checkAddDict(data)) {
+            let msg = "title,site,spidercode\nhref,channel,toptype\nsubtype,area\n非空必填信息,请补充完善信息"
+            showMsg(msg)
+            return;
+        }
+        var detail = document.getElementById('jsonDetail').value
+        var tmp_detail = ''+detail
+        if (tmp_detail=="") {
+            let msg = "请填写detail"
+            showMsg(msg)
+            return;
+        }
+
+        var tmp_content = document.getElementById('jsonContentHtml').value
+        if (tmp_content=="") {
+            let msg = "请填写contenthtml"
+            showMsg(msg)
+            return;
+        }
+
+        var modifyinfo = {}
+        var updata = {}
+        // 删除、修改
+        for (let editDataKey in edit_data) {
+            if (typeof edit_data[editDataKey] == "object") {
+                if (JSON.stringify(data[editDataKey]) != JSON.stringify(edit_data[editDataKey])) {
+                    modifyinfo[editDataKey] = "剑鱼维护"
+                    updata[editDataKey] = data[editDataKey]
+                }
+            }else {
+                if (data[editDataKey] != edit_data[editDataKey]) {
+                    modifyinfo[editDataKey] = "剑鱼维护"
+                    updata[editDataKey] = data[editDataKey]
+
+                }
+            }
+        }
+        //新增
+        for (let dataKey in data) {
+            if (edit_data[dataKey] == undefined) {
+                modifyinfo[dataKey] = "剑鱼维护"
+                updata[dataKey] = data[dataKey]
+            }
+        }
+        if (edit_detail != tmp_detail) {
+            modifyinfo["detail"] = "剑鱼维护"
+            updata["detail"] = tmp_detail
+        }
+
+        if (edit_contentHtml.replace(/(\s|\t|\n)/g,'') != tmp_content.replace(/(\s|\t|\n)/g,'')) {
+            modifyinfo["contenthtml"] = "剑鱼维护"
+            updata["contenthtml"] = tmp_content
+        }
+        var tmp_summary = $('#summary').val();
+        if (edit_summary != tmp_summary) {
+            modifyinfo["summary"] = "剑鱼维护"
+            updata["summary"] = tmp_summary
+        }
+
+        var str = ""
+        console.log(JSON.stringify(updata))
+        if (Object.keys(updata).length == 0) {
+            str = "没有修改任何字段信息！"
+            showTip(str)
+            return
+        }else {
+            str = "确定修改以下字段信息？" + "<br>"
+            for (const modifyinfoKey in modifyinfo) {
+                str = str + modifyinfoKey + "，"
+            }
+        }
+        showConfirm(str, function() {
+            $.ajax({
+                url: "/service/jy/save",
+                type: 'POST',
+                data: {"data": JSON.stringify(updata),
+                    "modifyinfo": JSON.stringify(modifyinfo),
+                    "id": sid,
+                    "coll": edit_coll,
+                    "reasons": reason,
+                },
+                success: function (task) {
+                    if (task.rep) {
+                        window.location.href="/service/jy/repair"
+                    } else {
+                        showTip("更新失败，请稍后在试");
+                    }
+                }
+            })
+        });
+    }
+
+    function checkAddDict(data) {
+        if (data["title"]==""||data["site"]==""||data["spidercode"]==""||
+            data["href"]==""||data["channel"]==""|| data["toptype"]==""||
+            data["subtype"]==""|| data["area"]==""){
+            return true
+        }
+        return false
+    }
+
+</script>