Merge branch 'master' of http://192.168.3.207:8080/data_processing/DataMaintenance

* 'master' of http://192.168.3.207:8080/data_processing/DataMaintenance:
  爬虫数据维护

src/config.json

@@ -53,6 +53,11 @@
     "port": 1482,
     "memo": "修改项目"
   },
+  "lua_db": {
+    "addr": "192.168.3.207:27092",
+    "dbsize": 5,
+    "dbname": "spider"
+  },
   "indexNode": {
     "addr": "127.0.0.1",
     "port": 1483,

src/filter/filter.go

@@ -10,7 +10,7 @@ import (
 func init() {
 	log.Println("过滤器")
 	matchUrl := make([]*regexp.Regexp, 0)
-	filter, _ := regexp.Compile("/(front|service)")
+	filter, _ := regexp.Compile("/(front|service|lua)")
 	matchUrl = append(matchUrl, filter)
 	xweb.AddFilter(&sessfilter{App: xweb.RootApp(), SessionName: "user",
 		MatchUrl: matchUrl})

src/lua/spiderwarn.go

@@ -0,0 +1,256 @@
+package lua
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/go-xweb/xweb"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"mongodb"
+	qu "qfw/util"
+	"strings"
+	"time"
+	. "util"
+)
+
+/*
+	spider_task程序统计spider_warn表中异常数据，并做同一条数据不同异常信息过滤入spider_warn_err表
+*/
+
+type Lua struct {
+	*xweb.Action
+	//spider_warn
+	spiderWarn xweb.Mapper `xweb:"/lua/spiderwarn"` //spider_warn异常数据
+	spiderEdit xweb.Mapper `xweb:"/lua/spideredit"` //编辑异常数据
+	spiderSave xweb.Mapper `xweb:"/lua/spiersave"`  //数据更新、保存
+
+}
+
+func (l *Lua) SpiderWarn() {
+	defer qu.Catch()
+	if l.Method() == "POST" {
+		defer qu.Catch()
+		level, _ := l.GetInteger("level")
+		start, _ := l.GetInteger("start")
+		limit, _ := l.GetInteger("length")
+		draw, _ := l.GetInteger("draw")
+		startTime, _ := l.GetInt("starttime")
+		qu.Debug(startTime, start, limit, draw, level)
+		sort := `{"%s":%d}`
+		orderIndex := l.GetString("order[0][column]")
+		orderName := l.GetString(fmt.Sprintf("columns[%s][data]", orderIndex))
+		orderType := 1
+		if l.GetString("order[0][dir]") != "asc" {
+			orderType = -1
+		}
+		sort = fmt.Sprintf(sort, orderName, orderType)
+		query := map[string]interface{}{
+			"ok": false,
+		}
+		if startTime > 0 {
+			query["comeintime"] = map[string]interface{}{
+				"$gte": startTime,
+				"$lt":  startTime + 86400,
+			}
+		}
+		if level != -1 {
+			query["level"] = level
+		}
+		fields := map[string]interface{}{
+			"data": 0,
+		}
+		count := MgoS.Count("spider_warn_err", query)
+		qu.Debug("query:", query, "sort:", sort, count)
+		list, _ := MgoS.Find("spider_warn_err", query, sort, fields, false, start, limit)
+		l.ServeJson(map[string]interface{}{
+			"data":            list,
+			"draw":            draw,
+			"recordsFiltered": count,
+			"recordsTotal":    count,
+		})
+	} else {
+		l.Render("lua/spiderwarn.html", &l.T)
+	}
+}
+
+func (l *Lua) SpiderEdit() {
+	defer qu.Catch()
+	id := l.GetString("id")
+	data := map[string]interface{}{}
+	bidId := ""
+	one, _ := MgoS.FindById("spider_warn_err", id, map[string]interface{}{"data": 1, "field": 1, "level": 1})
+	if one != nil && len(*one) > 0 {
+		data, _ = (*one)["data"].(map[string]interface{})
+		detail := qu.ObjToString((data)["detail"])
+		sha := Sha(detail)
+		contenthtml := qu.ObjToString((data)["contenthtml"])
+		summary := qu.ObjToString((data)["summary"])
+		data["s_sha"] = sha
+		//l.T["sha"] = sha
+		l.T["detail"] = detail
+		l.T["contenthtml"] = contenthtml
+		l.T["summary"] = summary
+		qu.Debug("id:", id, "	sha:", sha)
+		/*
+			1、如果field中出现一个以上字段表示spider_warn_err表在汇总spider_warn数据时出现一条数据多个异常信息(两个错误异常数据不可能是同条数据)
+			且该异常信息的原信息已经入bidding库，需要通过sha找bidding信息id
+			2、异常类型为警告时，原信息也已经入bidding库
+		*/
+		if qu.IntAll((*one)["level"]) == 1 || len(strings.Split(qu.ObjToString((*one)["field"]), ",")) > 1 {
+			bidData, _ := JYMgo.FindOne("bidding", map[string]interface{}{"s_sha": sha})
+			if bidData != nil && len(*bidData) > 0 {
+				bidId = mongodb.BsonIdToSId((*bidData)["_id"])
+				data = *bidData
+			} else {
+				qu.Debug("异常信息id:", id)
+				l.Render("com/err.html")
+				return
+			}
+		}
+		delete(data, "_id")
+		delete(data, "detail")
+		delete(data, "contenthtml")
+		delete(data, "summary")
+		delete(data, "_d")
+		delete(data, "T")
+		delete(data, "publishdept")
+		delete(data, "l_np_publishtime")
+	}
+	l.T["id"] = id     //spider_warn_err id
+	l.T["bid"] = bidId //bidding id
+	l.T["data"] = data
+	l.Render("lua/spideredit.html", &l.T)
+}
+
+func (l *Lua) SpiderSave() {
+	defer qu.Catch()
+	success := false
+	rep := ""
+	id := l.GetString("id")
+	stype, _ := l.GetInteger("stype")
+	qu.Debug("id:", id, "stype:", stype)
+	if stype == 2 { //无需更新：只在spider_warn_err上打标记
+		TagToSpiderWarnErr(stype, id, "无需修改", map[string]interface{}{})
+		success = true
+	} else {
+		bid := l.GetString("bid")
+		qu.Debug("bidding id:", bid)
+		reasons := l.GetString("reasons")
+		updateStr := l.GetStringComm("update")
+		update := map[string]interface{}{}
+		if err := json.Unmarshal([]byte(updateStr), &update); err != nil {
+			qu.Debug("data Unmarshal Failed：", err)
+		}
+		modifyStr := l.GetString("modifyinfo")
+		modifyinfo := map[string]interface{}{}
+		if err := json.Unmarshal([]byte(modifyStr), &modifyinfo); err != nil {
+			qu.Debug("modifyinfo Unmarshal Failed：", err)
+		}
+		if len(update) > 0 { //处理字段
+			for k, v := range update {
+				if k == "extracttype" || k == "dataging" {
+					(update)[k] = qu.IntAll(v)
+				} else if k == "publishtime" || k == "bidopentime" {
+					(update)[k] = qu.Int64All(v)
+				} else if k == "budget" || k == "bidamount" {
+					(update)[k] = qu.Float64All(v)
+				}
+			}
+			FormatNumber(update) //解决超大金额转成科学计数法的问题
+		}
+		qu.Debug(modifyinfo)
+		if stype == 1 { //修复更新
+			if bid == "" {
+				rep = "bidding id 为空"
+				goto L
+			}
+			old_data, _ := JYMgo.FindById("bidding", bid, nil) //查询原始信息，用作修改日志记录
+			if old_data != nil && len(*old_data) > 0 {
+				delete(*old_data, "_id")
+				if (*old_data)["modifyinfo"] != nil {
+					if m, ok := (*old_data)["modifyinfo"].(map[string]interface{}); ok { //合并modifyinfo
+						for k, v := range m {
+							modifyinfo[k] = v
+						}
+					}
+				}
+			} else {
+				rep = "未找到bidding数据,id:" + bid
+				goto L
+			}
+			if len(modifyinfo) > 0 { //记录修改字段
+				update["modifyinfo"] = modifyinfo
+			}
+			b := JYMgo.UpdateById("bidding", bid, map[string]interface{}{"$set": update}) //更新
+			if b {                                                                        //更新成功                                                                   //更新成功 重生bidding索引 项目合并
+				qu.Debug("更新成功")
+				//udptask.BiddingIndexUdp(bid, "bidding") //coll:bidding 是因为调用此方法的数据都是增量数据
+				//if IsModifyPro(modifyinfo) {            //修改了影响项目合并的字段 进行项目合并
+				//	udptask.ProjectSetUdp(bid, "bidding")
+				//}
+				//delName1 := RedisDelKey1 + bid + "_*"
+				//redis.DelByCodePattern(RedisJYName, delName1)
+				delete(update, "modifyinfo")
+				l.SaveUpdateLog(2, reasons, bid, *old_data, update, modifyinfo) //标签日志
+				TagToSpiderWarnErr(stype, id, reasons, update)                  //spider_warn_err日志
+				success = true
+			} else { //更新失败
+				rep = "更新bidding失败,id:" + bid
+				qu.Debug("更新失败")
+			}
+		} else { //3:修复发布 4:直接发布
+			href := qu.ObjToString(update["href"])
+			shaid := qu.ObjToString(update["s_sha"])
+			competehref := qu.ObjToString(update["competehref"]) //竞品数据
+			iscompete, _ := update["iscompete"].(bool)           //新爬虫数据
+			newId := primitive.NewObjectID()
+			newSid := mongodb.BsonIdToSId(newId)
+			qu.Debug("newId:", newSid)
+			update["_id"] = newId
+			if competehref != "" { //竞品爬虫
+				href = competehref
+				update["href"] = GetJyHref(newSid)
+			} else if iscompete { //新爬虫
+				update["competehref"] = "#"
+			}
+			PutHrefRedis(href, shaid, newSid)        //添加href进redis
+			update["comeintime"] = time.Now().Unix() //更新入库时间
+			qu.Debug(update)
+			JYMgo.SaveByOriID("bidding", update)                                                            //入bidding
+			l.SaveUpdateLog(1, reasons, newSid, map[string]interface{}{}, update, map[string]interface{}{}) //标签日志
+			TagToSpiderWarnErr(stype, id, reasons, update)                                                  //spider_warn_err日志
+			success = true
+		}
+	}
+L:
+	l.ServeJson(map[string]interface{}{"success": success, "rep": rep})
+}
+
+func TagToSpiderWarnErr(state int, id, reason string, update map[string]interface{}) {
+	defer qu.Catch()
+	set := map[string]interface{}{
+		"ok":         true,
+		"state":      state,
+		"reason":     reason,
+		"updatetime": time.Now().Unix(),
+	}
+	if len(update) > 0 {
+		set["update"] = update
+	}
+	MgoS.UpdateById("spider_warn_err", id, map[string]interface{}{"$set": set})
+}
+
+func (l *Lua) SaveUpdateLog(s_type int, reasons, bid string, old_data, update, modifyinfo map[string]interface{}) {
+	//日志记录
+	user := l.GetSession("user").(map[string]interface{})
+	log_data := map[string]interface{}{
+		"s_modifyuser":   user["name"],
+		"s_type":         s_type,
+		"i_modifytime":   time.Now().Unix(),
+		"s_modifyreason": reasons,
+		"s_backupid":     bid,
+		"o_oldinfo":      old_data,
+		"o_newinfo":      update,
+		"modifyinfo":     modifyinfo,
+	}
+	Mgo.Save(JyRecord, log_data)
+}

src/main.go

@@ -4,6 +4,7 @@ import (
 	_ "filter"
 	"front"
 	"log"
+	"lua"
 	qu "qfw/util"
 	"service"
 	"time"
@@ -32,6 +33,7 @@ func init() {
 	xweb.AddAction(&front.Front{})
 	xweb.AddAction(&service.RepairRule{})
 	xweb.AddAction(&service.OprdData{})
+	xweb.AddAction(&lua.Lua{})
 	xweb.RootApp().AppConfig.SessionTimeout = 1 * time.Hour
 	xweb.RootApp().Logger.SetOutputLevel(1)
 

src/service/repair_service.go

@@ -1,17 +1,14 @@
 package service
 
 import (
-	"crypto/sha256"
 	"encoding/json"
 	"fmt"
 	"github.com/go-xweb/xweb"
 	"github.com/tealeg/xlsx"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
-	"io"
 	"io/ioutil"
 	"log"
-	"math/big"
 	mu "mfw/util"
 	"mongodb"
 	"net"
@@ -261,34 +258,6 @@ func (jy *RepairRule) RepairPub() {
 	}
 }
 
-var reg = regexp.MustCompile("[^0-9A-Za-z\u4e00-\u9fa5]+")
-var h = sha256.New()
-var Filter = regexp.MustCompile("<[^>]*?>|[\\s\u3000\u2003\u00a0]")
-
-func Sha(con string) string {
-	h.Reset()
-	con = reg.ReplaceAllString(Filter.ReplaceAllString(con, ""), "")
-	h.Write([]byte(con))
-	return fmt.Sprintf("%x", h.Sum(nil))
-}
-
-func HexToBigIntMod(href string) int {
-	//取哈希值
-	t := sha256.New()
-	_, _ = io.WriteString(t, href)
-	hex := fmt.Sprintf("%x", t.Sum(nil))
-	//取模
-	n := new(big.Int)
-	n, _ = n.SetString(hex[2:], 16)
-	return int(n.Mod(n, big.NewInt(16)).Int64())
-}
-
-func HexText(href string) string {
-	h := sha256.New()
-	h.Write([]byte(href))
-	return fmt.Sprintf("%x", h.Sum(nil))
-}
-
 //编辑
 func (jy *RepairRule) RepairEdit() {
 	defer qu.Catch()
@@ -516,7 +485,7 @@ func (jy *RepairRule) RepairSave() {
 			}
 			qu.Debug("udp---1---------", string(by))
 			udptask.Udpclient.WriteUdp(by, mu.OP_TYPE_DATA, addr)
-			if isModifyPro(modifyinfo) {
+			if IsModifyPro(modifyinfo) {
 				// udp	项目
 				nextNode := *qu.ObjToMap(Sysconfig["jy_pro_node"])
 				by1, _ := json.Marshal(map[string]interface{}{
@@ -898,7 +867,7 @@ func ModifyData(tmp map[string]interface{}, user map[string]interface{}) (err ma
 	}
 	qu.Debug("udp------------", string(by))
 	udptask.Udpclient.WriteUdp(by, mu.OP_TYPE_DATA, addr)
-	if isModifyPro(modifyinfo) {
+	if IsModifyPro(modifyinfo) {
 		// udp	项目
 		nextNode := *qu.ObjToMap(Sysconfig["jy_pro_node"])
 		by1, _ := json.Marshal(map[string]interface{}{
@@ -1069,39 +1038,3 @@ func ModifyData1(tmp map[string]interface{}, user map[string]interface{}) (err m
 	Mgo.Save(JyRecord, log_data)
 	return nil
 }
-
-// 公告信息修改字段，需要修改项目信息的字段
-var ElementsPro = []string{
-	"projectname",
-	"projectcode",
-	"buyer",
-	"agency",
-	"area",
-	"city",
-	"publishtime",
-	"toptype",
-	"subtype",
-	"district",
-	"title",
-	"buyerperson",
-	"buyertel",
-	"buyerclass",
-	"createtime",
-	"review_experts",
-	"purchasing",
-	"href",
-	"topscopeclass",
-	"budget",
-	"bidamount",
-	"winner",
-	"s_winner",
-}
-
-func isModifyPro(tmp map[string]interface{}) bool {
-	for _, v := range ElementsPro {
-		if tmp[v] != nil {
-			return true
-		}
-	}
-	return false
-}

src/udptask/udptask.go

@@ -2,8 +2,10 @@ package udptask
 
 import (
 	"encoding/json"
+	"go.mongodb.org/mongo-driver/bson"
 	"log"
 	mu "mfw/util"
+	"mongodb"
 	"net"
 	qu "qfw/util"
 	"util"
@@ -50,3 +52,37 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 	}
 
 }
+
+//bidding索引udp
+func BiddingIndexUdp(id, coll string) {
+	indexNode := util.Sysconfig["indexNode"].(map[string]interface{})
+	addr := &net.UDPAddr{
+		IP:   net.ParseIP(indexNode["addr"].(string)),
+		Port: qu.IntAll(indexNode["port"]),
+	}
+	by, _ := json.Marshal(map[string]interface{}{
+		"query": map[string]interface{}{
+			"_id": bson.M{
+				"$gte": mongodb.StringTOBsonId(id),
+				"$lte": mongodb.StringTOBsonId(id),
+			}},
+		"stype": qu.ObjToString(indexNode["stype"]),
+		"coll":  coll,
+	})
+	Udpclient.WriteUdp(by, mu.OP_TYPE_DATA, addr)
+}
+
+//项目合并udp
+func ProjectSetUdp(id, coll string) {
+	nextNode := util.Sysconfig["jy_pro_node"].(map[string]interface{})
+	addr := &net.UDPAddr{
+		IP:   net.ParseIP(nextNode["addr"].(string)),
+		Port: qu.IntAll(nextNode["port"]),
+	}
+	by, _ := json.Marshal(map[string]interface{}{
+		"infoid": id,
+		"stype":  "updateInfo",
+		"coll":   coll,
+	})
+	Udpclient.WriteUdp(by, mu.OP_TYPE_DATA, addr)
+}

src/util/config.go

@@ -15,6 +15,7 @@ var (
 	OprdMgo         *mongodb.MongodbSim
 	MgoEn           *mongodb.MongodbSim
 	JyProMgo        *mongodb.MongodbSim
+	MgoS            *mongodb.MongodbSim
 	EsIndex, EsType string
 	MgoEnC          string
 	JyCollNameOne   string
@@ -58,6 +59,14 @@ func InitMgoPool() {
 		DbName:      qu.ObjToString(mgo["dbname"]),
 	}
 	OprdMgo.InitPool()
+	//lua
+	luaDb := *qu.ObjToMap(Sysconfig["lua_db"])
+	MgoS = &mongodb.MongodbSim{
+		MongodbAddr: qu.ObjToString(luaDb["addr"]),
+		Size:        qu.IntAll(luaDb["dbsize"]),
+		DbName:      qu.ObjToString(luaDb["dbname"]),
+	}
+	MgoS.InitPool()
 }
 
 func initJYMgo() {

src/util/util.go

@@ -1,14 +1,45 @@
 package util
 
 import (
+	"crypto/sha256"
+	"fmt"
+	"github.com/shopspring/decimal"
+	"io"
+	"math/big"
 	"net/http"
 	qu "qfw/util"
 	"reflect"
+	"regexp"
 	"strconv"
-
-	"github.com/shopspring/decimal"
 )
 
+var reg = regexp.MustCompile("[^0-9A-Za-z\u4e00-\u9fa5]+")
+var Filter = regexp.MustCompile("<[^>]*?>|[\\s\u3000\u2003\u00a0]")
+
+func Sha(con string) string {
+	h := sha256.New()
+	con = reg.ReplaceAllString(Filter.ReplaceAllString(con, ""), "")
+	h.Write([]byte(con))
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
+func HexToBigIntMod(href string) int {
+	//取哈希值
+	t := sha256.New()
+	_, _ = io.WriteString(t, href)
+	hex := fmt.Sprintf("%x", t.Sum(nil))
+	//取模
+	n := new(big.Int)
+	n, _ = n.SetString(hex[2:], 16)
+	return int(n.Mod(n, big.NewInt(16)).Int64())
+}
+
+func HexText(href string) string {
+	h := sha256.New()
+	h.Write([]byte(href))
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
 //数据类型转换
 func GetPostForm(r *http.Request) map[string]interface{} {
 	val := map[string]interface{}{}
@@ -59,3 +90,50 @@ func FormatNumber(tmp map[string]interface{}) {
 		}
 	}
 }
+
+// 公告信息修改字段，需要修改项目信息的字段
+var ElementsPro = []string{
+	"projectname",
+	"projectcode",
+	"buyer",
+	"agency",
+	"area",
+	"city",
+	"publishtime",
+	"toptype",
+	"subtype",
+	"district",
+	"title",
+	"buyerperson",
+	"buyertel",
+	"buyerclass",
+	"createtime",
+	"review_experts",
+	"purchasing",
+	"href",
+	"topscopeclass",
+	"budget",
+	"bidamount",
+	"winner",
+	"s_winner",
+}
+
+func IsModifyPro(tmp map[string]interface{}) bool {
+	for _, v := range ElementsPro {
+		if tmp[v] != nil {
+			return true
+		}
+	}
+	return false
+}
+
+func GetJyHref(id string) string {
+	return `https://www.jianyu360.cn/article/content/` + qu.CommonEncodeArticle("content", id) + `.html`
+}
+
+func PutHrefRedis(href, shaid, id string) {
+	db := HexToBigIntMod(href)
+	hashHref := HexText(href)
+	PutRedis("title_repeat_fulljudgement", db, hashHref, id, -1)
+	PutRedis("shaid", 0, shaid, "", 5184000)
+}

src/web/res/datepicker/bootstrap-datepicker.zh-CN.min.js

@@ -0,0 +1 @@
+!function(a){a.fn.datepicker.dates["zh-CN"]={days:["星期日","星期一","星期二","星期三","星期四","星期五","星期六"],daysShort:["周日","周一","周二","周三","周四","周五","周六"],daysMin:["日","一","二","三","四","五","六"],months:["一月","二月","三月","四月","五月","六月","七月","八月","九月","十月","十一月","十二月"],monthsShort:["1月","2月","3月","4月","5月","6月","7月","8月","9月","10月","11月","12月"],today:"今天",monthsTitle:"选择月份",clear:"清除",format:"yyyy-mm-dd",titleFormat:"yyyy年mm月",weekStart:1}}(jQuery);

src/web/templates/com/err.html

@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Title</title>
+</head>
+<body>
+<html>
+数据错误，请联系管理员！
+</html>
+</body>
+</html>

src/web/templates/com/inc.html

@@ -19,6 +19,7 @@
     <link rel="stylesheet" href="/other/css/bootstrap-select.min.css">
   	<link rel="stylesheet" href="/other/css/other.css">
 	<link rel="stylesheet" href="/other/css/bootstrap-switch.min.css">
+	<link rel="stylesheet" href="/datepicker/bootstrap-datepicker.min.css">
   
 
 	<!-- HTML5 Shim and Respond.js IE8 support of HTML5 elements and media queries -->
@@ -42,6 +43,8 @@
   	<script src="/js/bootstrap-select.min.js"></script>
 	<script src="/js/clipboard.min.js"></script>
 	<script src="/js/bootstrap-switch.min.js"></script>
+	<script src="/datepicker/bootstrap-datepicker.min.js"></script>
+	<script src="/datepicker/bootstrap-datepicker.zh-CN.min.js"></script>
 </head>
 <body class="hold-transition skin-blue sidebar-mini" style="background: #222d32">
 <div class="wrapper"></div>

src/web/templates/lua/spideredit.html

@@ -0,0 +1,247 @@
+{{include "com/inc.html"}}
+<!-- Main Header -->
+{{include "com/header.html"}}
+<!-- Left side column. 权限菜单 -->
+{{include "com/menu.html"}}
+
+<style>
+    /* 方法1：设置textarea合适的宽高 */
+    #jsonTextarea {
+        float: left;
+        margin-right: 20px;
+        width: 40%;
+        height: 70vh;
+        outline: none;
+        padding: 5px;
+    }
+
+    /* 方法2：自定义高亮样式 */
+    #jsonPre {
+        float: left;
+        width: 40%;
+        height: 70vh;
+        outline: 1px solid #ccc;
+        padding: 5px;
+        overflow: scroll;
+    }
+</style>
+
+
+<div class="content-wrapper">
+    <section class="content-header">
+        <h1>
+            {{if ne .T.bid ""}}
+                修复数据
+            {{else}}
+                新增数据
+            {{end}}
+        </h1>
+        <ol class="breadcrumb">
+            <li><a href="#"><i class="fa fa-dashboard"></i> 首页</a></li>
+            <li><a href="/lua/spiderwarn"> 爬虫数据维护</a></li>
+            <li><a href="#"> 添加数据</a></li>
+        </ol>
+    </section>
+    <!-- Main content -->
+    <section class="content">
+        <div class="nav-tabs-custom">
+            <ul class="nav nav-tabs edit-step">
+                {{if ne .T.bid ""}}
+                    <button class="btn btn-primary btn-sm" style="float: right;margin-top: 7px;margin-right: 10px" onclick="updateOrSave(1)"><i class="fa fa-fw fa-file-text fa-lg"></i>修复更新</button>
+                    <button class="btn btn-primary btn-sm" style="float: right;margin-top: 7px;margin-right: 10px" onclick="updateOrSave(2)"><i class="fa fa-fw fa-file-text fa-lg"></i>无需更新</button>
+                {{else}}
+                    <button class="btn btn-primary btn-sm" style="float: right;margin-top: 7px;margin-right: 10px" onclick="updateOrSave(3)"><i class="fa fa-fw fa-file-text fa-lg"></i>修复发布</button>
+                    <button class="btn btn-primary btn-sm" style="float: right;margin-top: 7px;margin-right: 10px" onclick="updateOrSave(4)"><i class="fa fa-fw fa-file-text fa-lg"></i>直接发布</button>
+                {{end}}
+            </ul>
+            <div class="box-body">
+                <div class="form-group">
+                    <div class="col-sm-3">
+                        <input type="text" class="form-control" id="reasons" placeholder="请输入更新原因" required>
+                    </div>
+                </div>
+            </div>
+            <form class="form-horizontal">
+                <div class="box-body">
+                    <textarea id="jsonTextarea" style="width: 100%;height: 300px"></textarea>
+                </div>
+                <div class="box-body">
+                    <label class="col-sm-2 control-left"><span style="color:red;">* </span>请填写detail</label>
+                    <textarea id="jsonDetail" style="width: 100%;height: 300px;padding: 5px"></textarea>
+                    <label class="col-sm-2 control-left"><span style="color:red;">* </span>请填写contenthtml</label>
+                    <textarea id="jsonContentHtml" style="width: 100%;height: 300px;padding: 5px"></textarea>
+                </div>
+                <div class="box-body" id="summaryDiv">
+                    <label class="col-sm-2 control-left">请填写summary</label>
+                    <textarea id="summary" style="width: 100%;height: 200px;padding: 5px"></textarea>
+                </div>
+            </form>
+        </div>
+    </section>
+</div>
+
+{{include "com/footer.html"}}
+<script>
+    menuActive("spiderwarn");
+    var edit_data = {{.T.data}}
+    var edit_contentHtml = {{.T.contenthtml}}
+    var edit_detail = {{.T.detail}}
+    var edit_summary = {{.T.summary}}
+    var bid = {{.T.bid}}
+    var edit_data_tmp = {{.T.data}}//记录汇总数据
+    edit_data_tmp["detail"] = edit_detail
+    edit_data_tmp["contenthtml"] = edit_contentHtml
+    edit_data_tmp["summary"] = edit_summary
+    function parse(str) {
+        return JSON.stringify(str, null, "\t")
+    }
+    $('#jsonTextarea').val(parse(edit_data)); //其他属性
+    $('#jsonDetail').val(edit_detail); //其他属性
+    $('#jsonContentHtml').val(edit_contentHtml); //其他属性
+    $('#summary').val(edit_summary);
+
+    function updateOrSave(stype){
+        /*
+        * stype:
+        *   1:修复更新
+        *   2:无需更新
+        *   3:修复发布
+        *   4:直接发布
+        * */
+         if (stype == 1 || stype == 3){
+            var reason=$("#reasons").val()
+            if (reason=="") {
+                showMsg("请填写更新数据原因")
+                return;
+            }
+        }
+
+        var data = JSON.parse($("#jsonTextarea").val())
+        // 字段非空校验
+        if (checkAddDict(data)) {
+            let msg = "title,site,spidercode\nhref,channel,toptype\nsubtype,area\n非空必填信息,请补充完善信息"
+            showMsg(msg)
+            return;
+        }
+        var detail = $("#jsonDetail").val()
+        var tmp_detail = ''+detail
+        if (tmp_detail=="") {
+            showMsg("请填写detail")
+            return;
+        }
+        var tmp_content = $("#jsonContentHtml").val()
+        if (tmp_content=="") {
+            showMsg("请填写contenthtml")
+            return;
+        }
+
+        // 判断除detail、contenthtml、summary外字段是否修改
+        var modifyinfo = {}
+        var update = {}
+        //  修改
+        for (var editDataKey in edit_data) {
+            if (typeof edit_data[editDataKey] == "object") {
+                if (JSON.stringify(data[editDataKey]) != JSON.stringify(edit_data[editDataKey])) {
+                    modifyinfo[editDataKey] = "剑鱼维护"
+                    update[editDataKey] = data[editDataKey]
+                    edit_data_tmp[editDataKey] = data[editDataKey]
+                }
+            }else {
+                if (data[editDataKey] != edit_data[editDataKey]) {
+                    modifyinfo[editDataKey] = "剑鱼维护"
+                    update[editDataKey] = data[editDataKey]
+                    edit_data_tmp[editDataKey] = data[editDataKey]
+                }
+            }
+        }
+        //新增
+        for (var dataKey in data) {
+            if (edit_data[dataKey] == undefined) {
+                modifyinfo[dataKey] = "剑鱼维护"
+                update[dataKey] = data[dataKey]
+                edit_data_tmp[dataKey] = data[dataKey]
+            }
+        }
+
+        //判断detail、contenthtml、summary是否修改
+        if (edit_detail != tmp_detail) {
+            modifyinfo["detail"] = "剑鱼维护"
+            update["detail"] = tmp_detail
+            edit_data_tmp["detail"] = tmp_detail
+        }
+        if (edit_contentHtml.replace(/(\s|\t|\n)/g,'') != tmp_content.replace(/(\s|\t|\n)/g,'')) {
+            modifyinfo["contenthtml"] = "剑鱼维护"
+            update["contenthtml"] = tmp_content
+            edit_data_tmp["contenthtml"] = tmp_content
+        }
+        var tmp_summary = $('#summary').val();
+        if (edit_summary != tmp_summary) {
+            modifyinfo["summary"] = "剑鱼维护"
+            update["summary"] = tmp_summary
+            edit_data_tmp["summary"] = tmp_summary
+        }
+
+        if (stype ==3 || stype ==4){//修复发布、直接发布 save
+            update = edit_data_tmp
+        }
+        var str = ""
+        if (Object.keys(update).length == 0 && stype != 2) {
+            str = "没有修改任何字段信息！"
+            showTip(str)
+            return
+        }else {
+            if(Object.keys(modifyinfo).length == 0){
+                str = "确定发布？"
+                if (stype ==3){
+                    showTip("没有修改任何字段信息！")
+                    return
+                }
+            }else{
+                if (stype ==2){
+                    showTip("请选择修复更新！")
+                    return
+                }
+                str = "确定修改以下字段信息？" + "<br>"
+                for (const modifyinfoKey in modifyinfo) {
+                    str = str + modifyinfoKey + "，"
+                }
+            }
+        }
+        console.log("---",update)
+        console.log("---",modifyinfo)
+        showConfirm(str, function() {
+            $.ajax({
+                url: "/lua/spiersave",
+                type: 'POST',
+                data: {
+                    "update": JSON.stringify(update),
+                    "modifyinfo": JSON.stringify(modifyinfo),
+                    "bid": bid,
+                    "id": {{.T.id}},
+                    "reasons": reason,
+                    "stype": stype
+                },
+                success: function (r) {
+                    if (r.success) {
+                        showMsg("保存成功",function (){
+                            window.location.href="/lua/spiderwarn"
+                        })
+                    } else {
+                        showTip(r.rep);
+                    }
+                }
+            })
+        });
+    }
+
+    function checkAddDict(data) {
+        if (data["title"]==""||data["site"]==""||data["spidercode"]==""||
+            data["href"]==""||data["channel"]==""|| data["toptype"]==""||
+            data["subtype"]==""|| data["area"]==""){
+            return true
+        }
+        return false
+    }
+
+
+</script>

src/web/templates/lua/spiderwarn.html

@@ -0,0 +1,193 @@
+{{include "com/inc.html"}}
+<!-- Main Header -->
+{{include "com/header.html"}}
+<!-- Left side column. 权限菜单 -->
+{{include "com/menu.html"}}
+
+<!-- Content Wrapper. Contains page content -->
+<div class="content-wrapper">
+    <section class="content-header">
+        <h1>
+            <small>
+<!--                <button type="button" class="btn btn-primary" data-toggle="modal" onclick="add()">新增用户</button>-->
+            </small>
+        </h1>
+        <ol class="breadcrumb">
+            <li><a href="/front/user"><i class="fa fa-dashboard"></i> 人员管理</a></li>
+        </ol>
+    </section>
+    <!-- Main content -->
+    <section class="content">
+        <div class="row">
+            <div class="col-xs-12">
+                <div class="box">
+                    <div class="box-body">
+                        <div>
+                            <div id="status-div" style="width: 205px;float: right">
+                                <span class="input-group date date-picker" id="starttime" data-provide="datepicker">
+                                    <input type="text" class="form-control form-filter input-sm" readonly name="starttime" placeholder="开始日期" />
+                                     <span class="input-group-addon">
+                                         <i class="fa fa-calendar"></i>
+                                      </span>
+                                </span>
+                            </div>
+                            <div style="display: flex;float: right;margin-right: 10px;align-items: center">
+                                <label style="white-space: nowrap;margin-bottom: 0;" for='name'>异常类型：</label>
+                                <select id='errtype' onchange='checkclick(this.value)' class='form-control input-sm'>
+                                    <option value='-1'>全部</option>
+                                    <option value='1'>警告</option>
+                                    <option value='2'>错误</option>
+                                </select>
+                            </div>
+
+                        </div>
+
+                        <table id="spiderwarn" class="table table-bordered table-hover">
+                            <thead>
+                            <tr>
+                                <th>编号</th>
+                                <th>爬虫</th>
+                                <th>站点</th>
+                                <th>类型</th>
+                                <th>字段</th>
+                                <th>错误原因</th>
+                                <th>操作</th>
+                            </tr>
+                            </thead>
+                        </table>
+                    </div>
+                    <!-- /.box-body -->
+                </div>
+                <!-- /.box -->
+            </div>
+        </div>
+    </section>
+</div>
+
+<div class="modal fade" id="modal-info">
+    <div class="modal-dialog">
+        <form id="userform" class="form-horizontal" role="form">
+            <div class="modal-content">
+                <div class="modal-header">
+                    <button type="button" class="close" data-dismiss="modal" aria-label="Close">
+                        <span aria-hidden="true">&times;</span></button>
+                    <h4 class="modal-title">用户信息</h4>
+                </div>
+                <div class="modal-body">
+                    <div class="form-group">
+                        <label for="code" class="col-sm-2 control-label">账号:</label>
+                        <div class="col-sm-10">
+                            <input id="email" name="email" type="email" class="form-control" placeholder="请输入账号">
+                        </div>
+                    </div>
+                    <div class="form-group">
+                        <label for="code" class="col-sm-2 control-label">密码:</label>
+                        <div class="col-sm-10">
+                            <input id="pwd" name="pwd" type="password" class="form-control" placeholder="请输入密码">
+                        </div>
+                    </div>
+                    <div class="form-group">
+                        <label for="site" class="col-sm-2 control-label">姓名:</label>
+                        <div class="col-sm-10">
+                            <input id="name" name="name" type="text" class="form-control" placeholder="请输入姓名">
+                        </div>
+                    </div>
+                    <div class="form-group">
+                        <label for="modify" class="col-sm-2 control-label">角色:</label>
+                        <div class="col-sm-10">
+                            <select id="role_select" class="form-control"></select>
+                        </div>
+                    </div>
+                </div>
+                <div class="modal-footer">
+                    <button type="button" class="btn btn-default" data-dismiss="modal">取消</button>
+                    <button type="button" class="btn btn-primary" onclick="save()">保存</button>
+                </div>
+            </div>
+            <!-- /.modal-content -->
+        </form>
+    </div>
+    <!-- /.modal-dialog -->
+</div>
+<!-- /.modal -->
+
+<!-- footer -->
+{{include "com/footer.html"}}
+
+<script>
+    menuActive("spiderwarn");
+    $(function () {
+        ttable = $('#spiderwarn').DataTable({
+            "ajax": {
+                "url": "/lua/spiderwarn",
+                "type": "post",
+                "data": {}
+            },
+            "language": {
+                "url": "/dist/js/dataTables.chinese.lang"
+            },
+            "columnDefs": [
+                { "orderable": false, "targets": [0,2,5,6] } //设置列不可排序
+            ],
+            "order": [[1,"desc"]],
+            "paging": true,
+            "lengthChange": false,
+            "searching": false,
+            "ordering": true,
+            "info": true,
+            "serverSide": true,
+            "autoWidth":true,
+            "columns": [
+                {
+                    "data": "_id", render: function (val, a, row) {
+                        return val="<a href='"+row.href+"' title='"+row.title+"' target='_blank'>"+row._id+"</a>"
+                    }
+                },
+                {"data": "code"},
+                {"data": "site"},
+                {"data": "level", render: function (val, a, row) {
+                    if(val == 2 ){
+                        return "错误";
+                    }
+                    return "警告";
+                }},
+                {"data": "field"},
+                {"data": "info"},
+                {
+                    "data": "_id", render: function (val, a, row) {
+                        return "<a href='/lua/spideredit?id="+val+"'><i class='fa fa-fw fa-edit text-yellow'></i></a>";
+                    }
+                }
+            ],
+            "fnServerParams": function (e) {
+                let st = $('#starttime').datepicker('getDate')
+                if (st != null) {
+                    let s = st.toLocaleDateString()
+                    let sc = Date.parse(new Date(s).toString())/1000
+                    e.starttime = sc
+                }
+                let level = $("#errtype").val();
+                if(level){
+                    e.level = level;
+                }else{
+                    e.level= -1;
+                }
+
+            }
+        });
+        $('.date-picker').datepicker({
+            language: 'zh-CN',
+            autoclose: true,
+            clearBtn: true, //清除按钮
+            todayBtn: false, //今日按钮
+            format: "yyyy-mm-dd"
+        });
+        $('#starttime').datepicker('setDate',new Date().toLocaleDateString());//设置初始时间
+        $('#starttime').datepicker().on('changeDate', function (e) {
+            ttable.ajax.reload();
+        })
+    })
+    function checkclick(){
+        ttable.ajax.reload();
+    }
+</script>