Merge branch 'master' of http://192.168.3.207:8080/data_processing/DataMaintenance

src/config.json

@@ -9,7 +9,6 @@
   "mgodben": "192.168.3.207:27092",
   "dbnameen": "enterprise",
   "mongoenc": "winner_enterprise",
-  "elasticsearchxs": "http://172.17.145.170:9800",
   "elasticsearch": "http://192.168.3.128:9800",
   "elasticsearch_index": "bidding",
   "elasticsearch_type": "bidding",

src/service/oprd_service.go

@@ -21,7 +21,7 @@ type OprdData struct {
 
 var Field = map[string]interface{}{"city": 1, "district": 1, "title": 1, "publishtime": 1, "projectname": 1, "projectcode": 1,
 	"budget": 1, "bidamount": 1, "bidopentime": 1, "buyer": 1, "buyertagname": 1, "buyerperson": 1, "buyertel": 1, "agency": 1,
-	"s_winner": 1, "winnerperson": 1, "winnertel": 1}
+	"s_winner": 1, "winnerperson": 1, "winnertel": 1, "subtype": 1, "toptype": 1}
 
 func (o *OprdData) ModifyData() {
 	defer qu.Catch()
@@ -76,7 +76,7 @@ func (o *OprdData) DelData() {
 			"msg": "id: " + id + "，错误！",
 		})
 	} else {
-		info, _ := util.Mgo.FindById("oprd_bidding", id, nil)
+		info, _ := util.OprdMgo.FindById("oprd_bidding", id, nil)
 		delete(*info, "_id")
 		user := o.GetSession("user").(map[string]interface{})
 		log_data := map[string]interface{}{
@@ -110,16 +110,27 @@ func (o *OprdData) SaveData() {
 	id := o.GetString("id")
 	old_data, _ := util.OprdMgo.FindById("oprd_bidding", id, Field)
 	delete(*old_data, "_id")
+	delMap := make(map[string]interface{})
 	for k, v := range updata {
-		if k == "publishtime" || k == "bidopentime" {
+		if v == "del" {
+			delMap[k] = ""
+			delete(updata, k)
+		} else if k == "publishtime" || k == "bidopentime" {
 			(updata)[k] = qu.IntAll(v)
 		} else if k == "budget" || k == "bidamount" {
 			(updata)[k] = qu.Float64All(v)
 		}
 	}
 	util.FormatNumber(updata)
-	qu.Debug(id, updata)
-	rep := util.OprdMgo.UpdateById("oprd_bidding", id, map[string]interface{}{"$set": updata})
+	updataMap := make(map[string]interface{})
+	if len(delMap) > 0 {
+		updataMap["$unset"] = delMap
+	}
+	if len(updata) > 0 {
+		updataMap["$set"] = updata
+	}
+	qu.Debug(id, updataMap)
+	rep := util.OprdMgo.UpdateById("oprd_bidding", id, updataMap)
 	if !rep {
 		o.ServeJson(map[string]interface{}{
 			"rep": rep,
@@ -132,6 +143,15 @@ func (o *OprdData) SaveData() {
 	if err != nil {
 		qu.Debug("delete es err:", err)
 	}
+	user := o.GetSession("user").(map[string]interface{})
+	log_data := map[string]interface{}{
+		"s_modifyuser":   user["name"],
+		"i_modifytime":   time.Now().Unix(),
+		"s_modifyreason": "修改",
+		"o_oldinfo":      *old_data,
+		"o_newinfo":      updata,
+	}
+	util.OprdMgo.Save("oprd_log", log_data)
 	o.ServeJson(map[string]interface{}{
 		"rep": rep,
 	})

src/service/repair_service.go

@@ -53,11 +53,27 @@ type RepairRule struct {
 
 	repairNewAdd xweb.Mapper `xweb:"/service/jy/repair/newAdd"`
 	repairPub    xweb.Mapper `xweb:"/service/jy/repair/pubSave"`
+
+	redisRepair xweb.Mapper `xweb:"/service/jy/repair/redis"`
+	esDel       xweb.Mapper `xweb:"/service/jy/repair/es/del"`
+	esDelRecord xweb.Mapper `xweb:"/service/jy/repair/es/del/record"`
+	esCount     xweb.Mapper `xweb:"/service/jy/repair/es/count/byField"`
+	esDelBy     xweb.Mapper `xweb:"/service/jy/repair/es/del/byField"`
 }
 
 func (jy *RepairRule) RepairList() {
 	defer qu.Catch()
-	jy.Render("repair/jy_repair.html")
+	_ = jy.Render("repair/jy_repair.html")
+}
+
+func (jy *RepairRule) RedisRepair() {
+	defer qu.Catch()
+	_ = jy.Render("repair/jy_redis_repair.html")
+}
+
+func (jy *RepairRule) EsDel() {
+	defer qu.Catch()
+	_ = jy.Render("repair/jy_es_del.html")
 }
 
 //新增数据
@@ -1040,3 +1056,54 @@ func ModifyData1(tmp map[string]interface{}, user map[string]interface{}) (err m
 	Mgo.Save(JyRecord, log_data)
 	return nil
 }
+
+func (jy *RepairRule) EsCount() {
+	defer qu.Catch()
+	value := jy.GetString("data")
+	field := jy.GetString("field")
+	esquery := `{"query":{"bool":{"must":[{"term":{"` + field + `":"` + value + `"}}]}}}`
+	count := elastic.Count(EsIndex, EsType, esquery)
+	jy.ServeJson(map[string]interface{}{
+		"rep":   true,
+		"count": count,
+	})
+}
+
+func (jy *RepairRule) EsDelBy() {
+	defer qu.Catch()
+	value := jy.GetString("data")
+	field := jy.GetString("field")
+	count, _ := jy.GetInt("count")
+	esquery := `{"query":{"bool":{"must":[{"term":{"` + field + `":"` + value + `"}}]}}}`
+	bol := elastic.Del(EsIndex, EsType, esquery)
+	Mgo.Save("jy_es_del_log", bson.M{"type": field, "value": value, "count": count, "createtime": time.Now().Unix()})
+	jy.ServeJson(map[string]interface{}{
+		"rep": bol,
+	})
+}
+
+func (jy *RepairRule) EsDelRecord() {
+	defer qu.Catch()
+	if jy.Method() == "POST" {
+		start, _ := jy.GetInteger("start")
+		limit, _ := jy.GetInteger("length")
+		draw, _ := jy.GetInteger("draw")
+		searchStr := jy.GetString("search[value]")
+		search := strings.TrimSpace(searchStr)
+		query := bson.M{}
+		if search != "" {
+			query["$or"] = []interface{}{
+				//bson.M{"s_customer": bson.M{"$regex": search}},
+				//bson.M{"s_tagname": bson.M{"$regex": search}},
+			}
+		}
+		data, _ := Mgo.Find("jy_es_del_log", query, `{"createtime": -1}`, nil, false, start, limit)
+		count := Mgo.Count("jy_es_del_log", query)
+		jy.ServeJson(map[string]interface{}{
+			"draw":            draw,
+			"data":            data,
+			"recordsFiltered": count,
+			"recordsTotal":    count,
+		})
+	}
+}

src/web/templates/oprd/bidding_modify.html

@@ -118,8 +118,11 @@
 
         var updata = {}
         // 删除、修改
+        console.log(edit_data["bidamount"], data["bidamount"])
         for (let editDataKey in edit_data) {
-            if (data[editDataKey] !== edit_data[editDataKey]) {
+            if (data[editDataKey] === undefined) {
+                updata[editDataKey] = "del"
+            }else if (data[editDataKey] !== edit_data[editDataKey]) {
                 updata[editDataKey] = data[editDataKey]
             }
         }

src/web/templates/repair/jy_es_del.html

@@ -0,0 +1,144 @@
+{{include "com/inc.html"}}
+<!-- Main Header -->
+{{include "com/header.html"}}
+<!-- Left side column. 权限菜单 -->
+{{include "com/menu.html"}}
+
+<div class="content-wrapper">
+    <section class="content-header">
+        <h1>es数据删除</h1>
+        <ol class="breadcrumb">
+            <li><a href="#"><i class="fa fa-dashboard"></i> 首页</a></li>
+            <li><a href="/service/jy/repair/es/del"> es数据删除</a></li>
+        </ol>
+    </section>
+    <!-- Main content -->
+    <section class="content">
+        <div class="nav-tabs-custom">
+            <form class="form-horizontal">
+                <br>
+                <div class="box-body">
+                    <div class="form-group">
+                        <label class="col-sm-2 control-label">爬虫代码</label>
+                        <div class="col-sm-3">
+                            <input type="text" class="form-control" id="spidercode" placeholder="爬虫代码" value="">
+                        </div>
+                        <input type="button" class="btn btn-danger" value="删除" onclick="delBy('spidercode')">
+                    </div>
+                    <div class="form-group">
+                        <label class="col-sm-2 control-label">站点名称</label>
+                        <div class="col-sm-3">
+                            <input type="text" class="form-control" id="site" placeholder="站点名称" value="">
+                        </div>
+                        <input type="button" class="btn btn-danger" value="删除" onclick="delBy('site')">
+                    </div>
+                    <br>
+                    <table id="dataTable" class="table table-bordered table-hover">
+                        <thead>
+                        <tr>
+                            <th>编号</th>
+                            <th>查询方式</th>
+                            <th>查询字段值</th>
+                            <th>删除条数</th>
+                            <th>操作时间</th>
+                        </tr>
+                        </thead>
+                    </table>
+                </div>
+            </form>
+        </div>
+    </section>
+</div>
+
+{{include "com/footer.html"}}
+<script>
+    menuActive("/service/jy/repair/es/del");
+
+    $(document).ready(function () {
+        ttable = $('#dataTable').dataTable({
+            "paging": true,
+            "lengthChange": false,
+            "searching": false,
+            "ordering": false,
+            "info": true,
+            "autoWidth": false,
+            "serverSide": false,
+            "ajax": {
+                "url": "/service/jy/repair/es/del/record",
+                "type": "post",
+                "data": {}
+            },
+            "language": {
+                "url": "/dist/js/dataTables.chinese.lang"
+            },
+            "fnDrawCallback": function () {
+                $("ul.pagination").prepend("&nbsp;&nbsp;&nbsp;转到第 <input type='text' id='changePage'   style='width:20px;'> 页    <a type='text' href='javascript:void(0);' id='dataTable-btn' style='text-align:center'>GO</a>");
+                $('#dataTable-btn').click(function (e) {
+                    var redirectpage = 0
+                    if ($("#changePage").val() && $("#changePage").val() > 0) {
+                        var redirectpage = $("#changePage").val() - 1;
+                    }
+                    ttable.page(redirectpage).draw(false);
+                });
+                this.api().column(0).nodes().each(function(cell, i) {
+                    cell.innerHTML = i + 1;
+                });
+            },
+            "columns": [
+                {"data": null},
+                {"data": "type"},
+                {"data": "value"},
+                {"data": "count"},
+                {"data": "createtime", render: function (val) {
+                        var dt = new Date()
+                        dt.setTime(parseInt(val) * 1000);
+                        return dt.format("yyyy-MM-dd hh:mm")
+                    }}
+            ]
+        });
+    });
+
+    function delBy(v) {
+        let value
+        if (v === "spidercode") {
+            value = $('#spidercode').val()
+            if (value === "") {
+                showMsg("爬虫代码不能为空")
+                return
+            }
+        }else if (v === "site") {
+            value = $('#site').val()
+            if (value === "") {
+                showMsg("站点名称不能为空")
+                return
+            }
+        }
+
+        $.ajax({
+            url: "/service/jy/repair/es/count/byField",
+            type: 'POST',
+            data: {"data": value, "field": v},
+            success: function (r) {
+                if (r.rep) {
+                    showConfirm("一共删除"+r.count+"条数据，确认需要删除吗？", function (){
+                        $.ajax({
+                            url: "/service/jy/repair/es/del/byField",
+                            type: 'POST',
+                            data: {"data": value, "field": v, "count": r.count},
+                            success: function (r) {
+                                if (r.rep) {
+                                    ttable.api().ajax.reload();
+                                }else {
+                                    showTip("删除失败");
+                                }
+                            }
+                        })
+                    })
+                } else {
+                    showTip("查询失败");
+                }
+            }
+        })
+    }
+
+</script>