重采平台项目提交

src/config.json

@@ -18,6 +18,13 @@
     "username": "",
     "password": ""
   },
+  "luaspiderdb": {
+    "addr": "192.168.3.207:27092",
+    "db": "spider",
+    "size": 5,
+    "username": "",
+    "password": ""
+  },
   "bideditor": {
     "addr": "192.168.3.207:27092",
     "db": "editor",

src/front/data.go

@@ -211,15 +211,7 @@ func (f *Front) DataUpdate() {
 		coll := f.GetString("coll")
 		id := f.GetString("id")
 		//字段类型转换
-		for k, v := range updateMap {
-			if k == "extracttype" {
-				(updateMap)[k] = qu.IntAll(v)
-			} else if strings.HasSuffix(k, "time") { //处理时间字段
-				(updateMap)[k] = qu.Int64All(v)
-			} else if k == "budget" || k == "bidamount" {
-				(updateMap)[k] = qu.Float64All(v)
-			}
-		}
+		util.FormatFields(updateMap)
 		util.FormatNumber(updateMap) //时间类型转换
 		query := map[string]interface{}{
 			"_id": mongodb.StringTOBsonId(id),
@@ -261,11 +253,12 @@ func (f *Front) DataUpdate() {
 				"T":        "bidding",
 			}
 			mustFiledNum := 0 //记录更新有效字段个数
-			for k, _ := range Fields {
+			for k, _ := range util.Fields {
 				if v := updateMap[k]; v != nil {
 					result[k] = v
 					mustFiledNum++
 				} else if v := (*data)[k]; v != nil {
+					mustFiledNum++
 					result[k] = v
 				}
 			}

src/front/front.go

@@ -56,6 +56,18 @@ type Front struct {
 	//data
 	importData xweb.Mapper `xweb:"/front/data/importdata"` //数据导入
 
+	//lua
+	warnList     xweb.Mapper `xweb:"/front/lua/warnlist"`     //spider_warn异常数据
+	warnEdit     xweb.Mapper `xweb:"/front/lua/warnedit"`     //编辑异常数据
+	uploadFile   xweb.Mapper `xweb:"/front/lua/uploadfile"`   //上传附件
+	luaDataSend  xweb.Mapper `xweb:"/front/lua/datasend"`     //数据更新、保存
+	spiderUpdate xweb.Mapper `xweb:"/front/lua/spiderupdate"` //批量更新状态
+
+	//data
+	//dataFind   xweb.Mapper `xweb:"/front/lua/datafind"`   //数据查找页面
+	//searchData xweb.Mapper `xweb:"/front/lua/searchdata"` //通过title、href查找
+	//dataView   xweb.Mapper `xweb:"/front/lua/dataview"`   //查看数据详情
+
 }
 
 func (f *Front) Login() {

src/front/lua.go

@@ -0,0 +1,336 @@
+package front
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"github.com/donnie4w/go-logger/logger"
+	"io/ioutil"
+	"mime/multipart"
+	qu "qfw/util"
+	. "spiderutil"
+	"strings"
+	. "task"
+	"time"
+	. "util"
+)
+
+/*
+	spider_task程序统计spider_warn表中异常数据，并做同一条数据不同异常信息过滤入spider_warn_err表
+*/
+
+func (f *Front) WarnList() {
+	defer qu.Catch()
+	if f.Method() == "POST" {
+		defer qu.Catch()
+		repeat, _ := f.GetInteger("repeat")
+		from := f.GetString("from")
+		start, _ := f.GetInteger("start")
+		limit, _ := f.GetInteger("length")
+		draw, _ := f.GetInteger("draw")
+		startTime, _ := f.GetInt("starttime")
+		searchStr := f.GetString("search[value]")
+		search := strings.TrimSpace(searchStr)
+		qu.Debug(startTime, start, limit, draw)
+		sort := `{"%s":%d}`
+		orderIndex := f.GetString("order[0][column]")
+		orderName := f.GetString(fmt.Sprintf("columns[%s][data]", orderIndex))
+		orderType := 1
+		if f.GetString("order[0][dir]") != "asc" {
+			orderType = -1
+		}
+		sort = fmt.Sprintf(sort, orderName, orderType)
+		query := map[string]interface{}{
+			"ok": false,
+		}
+		if repeat != -1 {
+			query["repeat"] = repeat == 1
+		}
+		if from != "-1" {
+			query["from"] = from
+		}
+		if startTime > 0 {
+			query["comeintime"] = map[string]interface{}{
+				"$gte": startTime,
+				"$lt":  startTime + 86400,
+			}
+		}
+		if search != "" {
+			query["$or"] = []interface{}{
+				map[string]interface{}{"site": map[string]interface{}{"$regex": search}},
+				map[string]interface{}{"code": map[string]interface{}{"$regex": search}},
+				map[string]interface{}{"title": map[string]interface{}{"$regex": search}},
+			}
+		}
+		fields := map[string]interface{}{
+			"data": 0,
+		}
+		count := MgoS.Count("spider_warn_err", query)
+		qu.Debug("query:", query, "sort:", sort, count)
+		// page := start / 10
+		list, _ := MgoS.Find("spider_warn_err", query, sort, fields, false, start, limit)
+		// for _, f := range *list {
+		// 	v["num"] = k + 1 + page*10
+		// }
+		f.ServeJson(map[string]interface{}{
+			"data":            list,
+			"draw":            draw,
+			"recordsFiltered": count,
+			"recordsTotal":    count,
+		})
+	} else {
+		f.Render("lua/warnlist.html", &f.T)
+	}
+}
+
+func (f *Front) WarnEdit() {
+	defer qu.Catch()
+	id := f.GetString("id")
+	num, _ := f.GetInteger("num")
+	data := map[string]interface{}{}
+	bidId := ""
+	one, _ := MgoS.FindById("spider_warn_err", id, map[string]interface{}{"data": 1, "field": 1})
+	if one != nil && len(*one) > 0 {
+		data, _ = (*one)["data"].(map[string]interface{})
+		detail := qu.ObjToString((data)["detail"])
+		contenthtml := qu.ObjToString((data)["contenthtml"])
+		summary := qu.ObjToString((data)["summary"])
+		f.T["detail"] = detail
+		f.T["contenthtml"] = contenthtml
+		f.T["summary"] = summary
+		delete(data, "_id")
+		delete(data, "detail")
+		delete(data, "contenthtml")
+		delete(data, "summary")
+		delete(data, "l_np_publishtime")
+	}
+	f.T["id"] = id     //spider_warn_err id
+	f.T["bid"] = bidId //bidding id
+	f.T["data"] = data
+	f.T["num"] = num
+	f.Render("lua/warnedit.html", &f.T)
+}
+
+func (f *Front) UploadFile() {
+	defer qu.Catch()
+	success := false
+	msg := ""
+	files := f.GetString("files")
+	id := f.GetString("id")
+	qu.Debug(id)
+	mf, err := f.GetFiles()
+	if err != nil {
+		msg = "附件上传失败"
+	} else {
+		fileMap := make(map[string]*multipart.File)
+		for _, hf := range mf {
+			f, _ := hf.Open()
+			fileMap[hf.Filename] = &f
+		}
+		var fileArr []map[string]interface{}
+		if err := json.Unmarshal([]byte(files), &fileArr); err != nil {
+			logger.Info("UserInfo Unmarshal Failed：", err)
+			msg = "附件解析失败"
+		} else {
+			var arr []map[string]interface{}
+			for _, m := range fileArr {
+				filename := qu.ObjToString(m["filename"])
+				org_url := qu.ObjToString(m["org_url"])
+				f := fileMap[filename]
+				if f == nil {
+					msg = "附件解析失败"
+					goto L
+				}
+				bt, _ := ioutil.ReadAll(*f)
+				key := GetHashKey(bt) + TypeByExt(filename)
+				b, _ := OssPutObject(key, bytes.NewReader(bt))
+				if b {
+					tmp := make(map[string]interface{})
+					tmp["org_url"] = org_url
+					tmp["filename"] = filename
+					tmp["ftype"] = strings.Replace(TypeByExt(filename), ".", "", -1)
+					tmp["fid"] = key
+					tmp["url"] = "oss"
+					br := bytes.NewReader(bt)
+					tmp["size"] = qu.ConvertFileSize(br.Len())
+					arr = append(arr, tmp)
+				} else {
+					msg = "文件上传oss失败"
+					goto L
+				}
+			}
+			if len(arr) > 0 {
+				attachments := map[string]interface{}{}
+				for i, tmp := range arr {
+					attachments[fmt.Sprint(i+1)] = tmp
+				}
+				success = MgoS.UpdateById("spider_warn_err", id, map[string]interface{}{"$set": map[string]interface{}{"data.projectinfo.attachments": attachments}})
+			}
+		}
+	}
+L:
+	f.ServeJson(map[string]interface{}{"success": success, "msg": msg})
+}
+
+func (f *Front) LuaDataSend() {
+	defer qu.Catch()
+	success := false
+	msg := ""
+	id := f.GetString("id")
+	stype, _ := f.GetInteger("stype")
+	qu.Debug("id:", id, "stype:", stype)
+	if stype == 2 { //无需更新：只在spider_warn_err上打标记
+		TagToSpiderWarnErr(stype, id, "无需修改或发布", map[string]interface{}{})
+		success = true
+	} else {
+		bid := f.GetString("bid")
+		qu.Debug("bidding id:", bid)
+		reasons := f.GetString("reasons")
+		updateStr := f.GetStringComm("update")
+		update := map[string]interface{}{}
+		if err := json.Unmarshal([]byte(updateStr), &update); err != nil {
+			qu.Debug("data Unmarshal Failed：", err)
+		}
+		//处理字段
+		if len(update) > 0 {
+			FormatFields(update)
+			FormatNumber(update) //解决超大金额转成科学计数法的问题
+		}
+		one, _ := MgoS.FindById("spider_warn_err", id, map[string]interface{}{"data": 1})
+		if len(*one) > 0 {
+			data, _ := (*one)["data"].(map[string]interface{})
+			mustFiledNum := 0 //记录更新有效字段个数
+			result := map[string]interface{}{}
+			for k, _ := range Fields {
+				if v := update[k]; v != nil {
+					result[k] = v
+					mustFiledNum++
+				} else if v := data[k]; v != nil {
+					mustFiledNum++
+					result[k] = v
+				}
+			}
+			if mustFiledNum > 0 {
+				flag, id, coll := SaveObj(4002, "title", result)
+				update["sendflag"] = flag
+				update["biddingid"] = id
+				update["biddingcoll"] = coll
+				TagToSpiderWarnErr(stype, id, reasons, update) //spider_warn_err日志
+				success = true
+			} else {
+				msg = "推送失败，无有效字段"
+			}
+		} else {
+			msg = "未找到该数据"
+		}
+		//if stype == 1 { //修复更新
+		//	if bid == "" {
+		//		rep = "bidding id 为空"
+		//		goto L
+		//	}
+		//	old_data, _ := JYMgo.FindById("bidding", bid, nil) //查询原始信息，用作修改日志记录
+		//	if old_data != nil && len(*old_data) > 0 {
+		//		delete(*old_data, "_id")
+		//		if (*old_data)["modifyinfo"] != nil {
+		//			if m, ok := (*old_data)["modifyinfo"].(map[string]interface{}); ok { //合并modifyinfo
+		//				for k, v := range m {
+		//					modifyinfo[k] = v
+		//				}
+		//			}
+		//		}
+		//	} else {
+		//		rep = "未找到bidding数据,id:" + bid
+		//		goto L
+		//	}
+		//	if len(modifyinfo) > 0 { //记录修改字段
+		//		update["modifyinfo"] = modifyinfo
+		//	}
+		//	b := JYMgo.UpdateById("bidding", bid, map[string]interface{}{"$set": update}) //更新
+		//	if b {                                                                        //更新成功                                                                   //更新成功 重生bidding索引 项目合并
+		//		qu.Debug("更新成功")
+		//		udptask.BiddingIndexUdp(bid, "bidding") //coll:bidding 是因为调用此方法的数据都是增量数据
+		//		if IsModifyPro(modifyinfo) {            //修改了影响项目合并的字段 进行项目合并
+		//			udptask.ProjectSetUdp(bid, "bidding")
+		//		}
+		//		delName1 := RedisDelKey1 + "*_" + bid
+		//		redis.DelByCodePattern(RedisJYName, delName1)
+		//		delete(update, "modifyinfo")
+		//		l.SaveUpdateLog(2, reasons, bid, *old_data, update, modifyinfo) //标签日志
+		//		TagToSpiderWarnErr(stype, id, reasons, update)                  //spider_warn_err日志
+		//		success = true
+		//	} else { //更新失败
+		//		rep = "更新bidding失败,id:" + bid
+		//		qu.Debug("更新失败")
+		//	}
+		//} else { //3:修复发布 4:直接发布
+		//	href := qu.ObjToString(update["href"])
+		//	shaid := qu.ObjToString(update["s_sha"])
+		//	competehref := qu.ObjToString(update["competehref"]) //竞品数据
+		//	iscompete, _ := update["iscompete"].(bool)           //新爬虫数据
+		//	newId := primitive.NewObjectID()
+		//	newSid := mongodb.BsonIdToSId(newId)
+		//	qu.Debug("newId:", newSid)
+		//	update["_id"] = newId
+		//	if competehref != "" { //竞品爬虫
+		//		href = competehref
+		//		update["href"] = GetJyHref(newSid)
+		//	} else if iscompete { //新爬虫
+		//		update["competehref"] = "#"
+		//	}
+		//	delete(update, "iscompete")
+		//	update["comeintime"] = time.Now().Unix()        //更新入库时间
+		//	update["modifyinfo"] = map[string]interface{}{} //添加空modifyinfo作为标识
+		//	PutHrefRedis(href, shaid, newSid)               //添加href进redis
+		//	qu.Debug(update)
+		//	JYMgo.SaveByOriID("bidding", update)                                                            //入bidding
+		//	l.SaveUpdateLog(1, reasons, newSid, map[string]interface{}{}, update, map[string]interface{}{}) //标签日志
+		//	TagToSpiderWarnErr(stype, id, reasons, update)                                                  //spider_warn_err日志
+		//	success = true
+		//}
+	}
+	f.ServeJson(map[string]interface{}{"success": success, "msg": msg})
+}
+
+func (f *Front) SpiderUpdate() {
+	defer qu.Catch()
+	state, _ := f.GetInteger("state")
+	reason := "批量处理无需更新"
+	if state == 4 { //批量直接发布
+		reason = "批量处理直接发布"
+	}
+	ids := strings.Split(f.GetString("ids"), ",")
+	for _, id := range ids {
+		TagToSpiderWarnErr(state, id, reason, map[string]interface{}{})
+	}
+	f.ServeJson(map[string]interface{}{"success": true})
+}
+
+func TagToSpiderWarnErr(state int, id, reason string, update map[string]interface{}) {
+	defer qu.Catch()
+	set := map[string]interface{}{
+		"ok":         true,
+		"state":      state,
+		"reason":     reason,
+		"updatetime": time.Now().Unix(),
+	}
+	if len(update) > 0 {
+		set["update"] = update
+	}
+	MgoS.UpdateById("spider_warn_err", id, map[string]interface{}{"$set": set})
+}
+
+//func (f *Front) SaveUpdateLog(s_type int, reasons, bid string, old_data, update, modifyinfo map[string]interface{}) {
+//	//日志记录
+//	user := f.GetSession("user").(map[string]interface{})
+//	log_data := map[string]interface{}{
+//		"s_modifyuser":   user["name"],
+//		"s_type":         s_type,
+//		"i_modifytime":   time.Now().Unix(),
+//		"s_modifyreason": reasons,
+//		"s_backupid":     bid,
+//		"o_oldinfo":      old_data,
+//		"o_newinfo":      update,
+//		"modifyinfo":     modifyinfo,
+//	}
+//	MgoS.Save(JyRecord, log_data)
+//}

src/task/task.go

@@ -16,25 +16,6 @@ import (
 	. "util"
 )
 
-var Fields = map[string]interface{}{
-	"publishtime": 1,
-	"detail":      1,
-	"contenthtml": 1,
-	"title":       1,
-	"href":        1,
-	"area":        1,
-	"city":        1,
-	"district":    1,
-	"site":        1,
-	"channel":     1,
-	"spidercode":  1,
-	"jsondata":    1,
-	"_d":          1,
-	"iscompete":   1, //由爬虫决定
-	"comeintime":  1,
-	//"publishdept": 1,
-	//"type":        1,
-}
 var RunningTask = map[string]bool{} //记录正在执行的任务所用的表
 var TaskLock = &sync.Mutex{}
 

src/util/config.go

@@ -13,6 +13,7 @@ var (
 	MgoDT    *mongodb.MongodbSim //数据存储库
 	MgoE     *mongodb.MongodbSim //爬虫库
 	MgoB     *mongodb.MongodbSim //bidding库
+	MgoS     *mongodb.MongodbSim //lua spider库
 	DataBb   string              //数据存储库
 	DataColl string              //数据存储表
 	FlowsArr []string            //数据处理流程
@@ -54,7 +55,7 @@ func InitMgoPool() {
 		Password:    Config.BidEditor.Password,
 	}
 	MgoE.InitPool()
-	//爬虫库
+	//bidding库
 	MgoB = &mongodb.MongodbSim{
 		MongodbAddr: Config.Bidding.Addr,
 		Size:        Config.Bidding.Size,
@@ -63,6 +64,13 @@ func InitMgoPool() {
 		Password:    Config.Bidding.Password,
 	}
 	MgoB.InitPool()
+	//lua spider库
+	MgoS = &mongodb.MongodbSim{
+		MongodbAddr: Config.LuaSpiderDB.Addr,
+		Size:        Config.LuaSpiderDB.Size,
+		DbName:      Config.LuaSpiderDB.Db,
+	}
+	MgoS.InitPool()
 	//
 	FlowsMap = map[string]*Flow{}
 	for _, flow := range Config.Flows {

src/util/util.go

@@ -5,10 +5,48 @@ import (
 	qu "qfw/util"
 	"reflect"
 	"strconv"
+	"strings"
 )
 
+var Fields = map[string]interface{}{
+	"publishtime": 1,
+	"detail":      1,
+	"contenthtml": 1,
+	"title":       1,
+	"href":        1,
+	"competehref": 1,
+	"area":        1,
+	"city":        1,
+	"district":    1,
+	"site":        1,
+	"channel":     1,
+	"spidercode":  1,
+	"jsondata":    1,
+	"_d":          1,
+	"iscompete":   1, //由爬虫决定
+	"comeintime":  1,
+	"projectinfo": 1,
+	"T":           1,
+	"dataging":    1,
+	//"publishdept": 1,
+	//"type":        1,
+}
+
 const GTEID = "5a862f0640d2d9bbe88e3cec"
 
+//处理因前后端交互，字段类型转换
+func FormatFields(tmp map[string]interface{}) {
+	for k, v := range tmp {
+		if k == "dataging" || k == "infoformat" {
+			tmp[k] = qu.IntAll(v)
+		} else if strings.HasSuffix(k, "time") { //处理时间字段
+			tmp[k] = qu.Int64All(v)
+		} else if k == "budget" || k == "bidamount" {
+			tmp[k] = qu.Float64All(v)
+		}
+	}
+}
+
 /**
  *	前端科学计数法处理
  */

src/web/templates/com/header.html

@@ -108,6 +108,27 @@
   	</div>
   <!-- /.modal-dialog -->
 </div>
+
+<!--loading-->
+<div class="modal fade" id="loading" tabindex="-1" role="dialog" aria-labelledby="myModalLabel" aria-hidden="true" data-backdrop="static" data-keyboard="false">>
+	<div class="modal-dialog">
+		<div class="modal-content" style="margin-top: 220px">
+			<div class="modal-header">
+				<div class="modal-header">
+					<div class="edit-form">
+						<form class="form-horizontal" role="form">
+							<div class="form-group">
+								<span id="loadText" class="info"></span>
+								<span class="fa fa-spin fa-spinner" aria-hidden="true"></span>
+							</div>
+						</form>
+					</div>
+				</div>
+			</div>
+		</div><!-- /.modal-content -->
+	</div><!-- /.modal -->
+</div>
+
 <style>
 
 
@@ -133,10 +154,10 @@ showLoading = function (text){
 	if(text){
 		$("#loadText").html(text)
 	}
-	$('#loadingModal').modal({backdrop: 'static', keyboard: false});
+	$('#loading').modal({backdrop: 'static', keyboard: false});
 }
 //隐藏掉loading框
 hideLoading = function (){
-	$('#loadingModal').modal('hide');
+	$('#loading').modal('hide');
 }
 </script>

src/web/templates/lua/warnedit.html

@@ -0,0 +1,361 @@
+{{include "com/inc.html"}}
+<!-- Main Header -->
+{{include "com/header.html"}}
+<!-- Left side column. 权限菜单 -->
+{{include "menu/menu.html"}}
+
+<style>
+    /* 方法1：设置textarea合适的宽高 */
+    #jsonTextarea {
+        float: left;
+        margin-right: 20px;
+        width: 40%;
+        height: 70vh;
+        outline: none;
+        padding: 5px;
+    }
+
+    /* 方法2：自定义高亮样式 */
+    #jsonPre {
+        float: left;
+        width: 40%;
+        height: 70vh;
+        outline: 1px solid #ccc;
+        padding: 5px;
+        overflow: scroll;
+    }
+</style>
+<div class="modal fade" id="uploadfile" tabindex="-1" role="dialog" aria-labelledby="myModalLabel" aria-hidden="true">
+    <div class="modal-dialog">
+        <div class="modal-content">
+            <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal" aria-label="Close">
+                    <span aria-hidden="true">&times;</span>
+                </button>
+                <h4>
+                    <span class="fa fa-cloud-upload" id="code-assign" aria-hidden="true"></span>
+                    <span class="info">附件</span>
+                </h4>
+
+            </div>
+            <div class="modal-body" style="max-height: 70vh;overflow-y: scroll;">
+                <form class="form-horizontal" role="form">
+                    <div id="TaskDiv">
+                        <div id="itemDiv" style="display: none">
+                            <div class="form-group">
+                                <label class="col-sm-2 control-label">附件地址:</label>
+                                <div class="col-sm-5">
+                                    <input type="text" class="furl form-control" placeholder="附件下载地址" value="">
+                                </div>
+                            </div>
+                            <div class="form-group">
+                                <label class="col-sm-2 control-label">本地文件:</label>
+                                <div class="col-sm-5">
+                                    <input type="file" name="file" class="ffile" id="uploadfile-add-0">
+                                </div>
+                                <div style="float: right;margin-right: 158px">
+                                    <input type="button" class="btn btn-info" value="删除" onclick="removeFile(this)">
+                                </div>
+                            </div>
+<!--                            <div class="form-group">-->
+<!--                                <label class="col-sm-2 control-label">附件名称:</label>-->
+<!--                                <div class="col-sm-5">-->
+<!--                                    <input type="text" class="fname form-control" placeholder="附件名称.类型" value="">-->
+<!--                                </div>-->
+<!--                            </div>-->
+                        </div>
+                    </div>
+                    <div class="group-item">
+                        <div class="form-group">
+                            <label class="col-sm-2 control-label">附件地址:</label>
+                            <div class="col-sm-5">
+                                <input type="text" class="furl form-control" id="downloadurl" placeholder="附件下载地址" value="">
+                            </div>
+                        </div>
+                        <div class="form-group">
+                            <label class="col-sm-2 control-label">本地文件:</label>
+                            <div class="col-sm-5">
+                                <input type="file" name="file" class="ffile" id="uploadfile-add">
+                            </div>
+                            <div style="float: right;margin-right: 100px">
+                                <input type="button" class="btn btn-info" value="新增" onclick="addFile()">
+                                <input type="button" class="btn btn-info" value="上传" onclick="uploadFile()">
+                            </div>
+                        </div>
+<!--                        <div class="form-group">-->
+<!--                            <label class="col-sm-2 control-label">附件名称:</label>-->
+<!--                            <div class="col-sm-5">-->
+<!--                                <input type="text" class="fname form-control" id="filename" placeholder="附件名称.类型" value="">-->
+<!--                            </div>-->
+<!--                            <input type="button" class="btn btn-info" value="新增" onclick="addFile()">-->
+<!--                            <input type="button" class="btn btn-info" value="上传" onclick="uploadFile()">-->
+<!--                        </div>-->
+                    </div>
+                </form>
+                <!--</div>	-->
+            </div>
+        </div>
+    </div>
+</div>
+
+<div class="content-wrapper">
+    <section class="content-header">
+        <h1>
+            {{if ne .T.bid ""}}
+                修复数据
+            {{else}}
+                新增数据
+            {{end}}
+        </h1>
+        <ol class="breadcrumb">
+            <li><a href="#"><i class="fa fa-dashboard"></i> 爬虫数据维护</a></li>
+            <li><a href="front/lua/warnlist"> 异常数据管理</a></li>
+            <li><a href="#">数据维护</a></li>
+        </ol>
+    </section>
+    <!-- Main content -->
+    <section class="content">
+        <div class="nav-tabs-custom">
+            <ul class="nav nav-tabs edit-step">
+                <button class="btn btn-primary btn-sm" style="float: left;margin-top: 7px;margin-left: 10px" onclick="showUploadFileModal()"><i class="fa fa-fw fa-file-text fa-lg"></i>补充附件</button>
+                {{if ne .T.bid ""}}
+                    <button class="btn btn-primary btn-sm" style="float: right;margin-top: 7px;margin-right: 10px" onclick="updateOrSave(1)">修复更新</button>
+                <button class="btn btn-primary btn-sm" style="float: right;margin-top: 7px;margin-right: 10px" onclick="updateOrSave(2)">无需更新</button>
+                {{else}}
+                    <button class="btn btn-primary btn-sm" style="float: right;margin-top: 7px;margin-right: 10px" onclick="updateOrSave(3)">修复发布</button>
+<!--                    <button class="btn btn-primary btn-sm" style="float: right;margin-top: 7px;margin-right: 10px" onclick="updateOrSave(4)">直接发布</button>-->
+                    <button class="btn btn-primary btn-sm" style="float: right;margin-top: 7px;margin-right: 10px" onclick="updateOrSave(2)">无需发布</button>
+                {{end}}
+
+            </ul>
+<!--            <div class="box-body">-->
+<!--                <div class="form-group">-->
+<!--                    <div class="col-sm-3">-->
+<!--                        <input type="text" class="form-control" id="reasons" placeholder="请输入更新原因" required>-->
+<!--                    </div>-->
+<!--                </div>-->
+<!--            </div>-->
+            <form class="form-horizontal">
+                <div class="box-body">
+                    <textarea id="jsonTextarea" style="width: 100%;height: 300px"></textarea>
+                </div>
+                <div class="box-body">
+                    <label class="col-sm-2 control-left"><span style="color:red;">* </span>请填写detail</label>
+                    <textarea id="jsonDetail" style="width: 100%;height: 300px;padding: 5px"></textarea>
+                    <label class="col-sm-2 control-left"><span style="color:red;">* </span>请填写contenthtml</label>
+                    <textarea id="jsonContentHtml" style="width: 100%;height: 300px;padding: 5px"></textarea>
+                </div>
+                <div class="box-body" id="summaryDiv">
+                    <label class="col-sm-2 control-left">请填写summary</label>
+                    <textarea id="summary" style="width: 100%;height: 200px;padding: 5px"></textarea>
+                </div>
+            </form>
+        </div>
+    </section>
+</div>
+
+{{include "com/footer.html"}}
+<script>
+    menuActive("warnlist");
+    var fileNameReg = new RegExp("(.jpg|.png|.gif|.pdf|.doc|.docx|.xlsx|.xls|.zip|.rar)$");
+    var edit_data = {{.T.data}}
+    var edit_contentHtml = {{.T.contenthtml}}
+    var edit_detail = {{.T.detail}}
+    var edit_summary = {{.T.summary}}
+    var bid = {{.T.bid}}
+    var edit_data_tmp = {{.T.data}}//记录汇总数据
+    edit_data_tmp["detail"] = edit_detail
+    edit_data_tmp["contenthtml"] = edit_contentHtml
+    edit_data_tmp["summary"] = edit_summary
+    function parse(str) {
+        return JSON.stringify(str, null, "\t")
+    }
+    $('#jsonTextarea').val(parse(edit_data)); //其他属性
+    $('#jsonDetail').val(edit_detail); //其他属性
+    $('#jsonContentHtml').val(edit_contentHtml); //其他属性
+    $('#summary').val(edit_summary);
+
+    function updateOrSave(stype){
+        /*
+        * stype:
+        *   1:修复更新
+        *   2:无需更新
+        *   3:修复发布
+        *   4:直接发布（现版本展示的都是错误数据，直接发布是无效的）
+        * */
+         if (stype == 1 || stype == 3){
+            var reason=$("#reasons").val()
+            if (reason=="") {
+                showMsg("请填写更新数据原因")
+                return;
+            }
+        }
+
+        var data = JSON.parse($("#jsonTextarea").val())
+        // 字段非空校验
+        // if (checkAddDict(data)) {
+        //     let msg = "title,site,spidercode\nhref,channel,toptype\nsubtype,area\n非空必填信息,请补充完善信息"
+        //     showMsg(msg)
+        //     return;
+        // }
+        var detail = $("#jsonDetail").val()
+        var tmp_detail = ''+detail
+        if (tmp_detail=="") {
+            showMsg("请填写detail")
+            return;
+        }
+        var tmp_content = $("#jsonContentHtml").val()
+        if (tmp_content=="") {
+            showMsg("请填写contenthtml")
+            return;
+        }
+
+        // 判断除detail、contenthtml、summary外字段是否修改
+        var modifyinfo = []
+        var update = {}
+        // 删除、修改
+        for (let editDataKey in edit_data) {
+            if (typeof edit_data[editDataKey] == "object") {
+                if (JSON.stringify(data[editDataKey]) != JSON.stringify(edit_data[editDataKey])) {
+                    modifyinfo.push(editDataKey)
+                    update[editDataKey] = data[editDataKey]
+                }
+            }else {
+                if (data[editDataKey] != edit_data[editDataKey]) {
+                    modifyinfo.push(editDataKey)
+                    update[editDataKey] = data[editDataKey]
+
+                }
+            }
+        }
+        //新增
+        for (let dataKey in data) {
+            if (edit_data[dataKey] == undefined) {
+                modifyinfo.push(dataKey)
+                update[dataKey] = data[dataKey]
+            }
+        }
+        if (edit_detail != tmp_detail) {
+            modifyinfo.push("detail")
+            update["detail"] = tmp_detail
+        }
+
+        if (edit_contentHtml.replace(/(\s|\t|\n)/g,'') != tmp_content.replace(/(\s|\t|\n)/g,'')) {
+            modifyinfo.push("contenthtml")
+            update["contenthtml"] = tmp_content
+        }
+        var tmp_summary = $('#summary').val();
+        if (edit_summary != tmp_summary) {
+            modifyinfo.push("summary")
+            update["summary"] = tmp_summary
+        }
+
+        var str = "确定操作？"
+        if (stype != 2 && {{.T.num}} != 0){
+            if (Object.keys(update).length == 0) {
+                str = "没有修改任何字段信息！"
+                showTip(str)
+                return
+            }else {
+                str = "确定修改以下字段信息？" + "<br>"+modifyinfo.join(",");
+            }
+        }
+        showConfirm(str, function() {
+            $.ajax({
+                url: "/front/lua/datasend",
+                type: 'POST',
+                data: {
+                    "update": JSON.stringify(update),
+                    "bid": bid,
+                    "id": {{.T.id}},
+                    "reasons": reason,
+                    "stype": stype
+                },
+                success: function (r) {
+                    if (r.success) {
+                        showMsg("保存成功",function (){
+                            window.location.href="/front/lua/warnlist"
+                        })
+                    } else {
+                        showTip(r.msg);
+                    }
+                }
+            })
+        });
+    }
+
+    function checkAddDict(data) {
+        if (data["title"]||data["site"]||data["spidercode"]||
+            data["href"]||data["channel"]||data["toptype"]||
+            data["subtype"]||data["area"]){
+            return true
+        }
+        return false
+    }
+
+    function showUploadFileModal() {
+        $('#uploadfile').modal("show");
+    }
+
+    function removeFile(e) {
+        $(e).parents('#itemDiv').remove()
+    }
+    
+    function addFile() {
+        let tNode = $('#itemDiv').clone().addClass('group-item').addClass('clone-template').show()
+        $('#TaskDiv').append($(tNode))
+    }
+
+    function uploadFile() {
+        var arr = []
+        var fData = new FormData();
+        try {
+            $('.group-item').each(function () {
+                var furl = $(this).find("input.furl").val()
+                var fFile = $(this).find('input.ffile')[0].files[0]
+                if (fFile === undefined) {
+                    throw "请选择本地文件";
+                }
+                var fName = $(this).find("input.ffile").val();
+                var nameArr = fName.split('\\');
+                fName = nameArr[nameArr.length-1];
+                if (!fileNameReg.test(fName.toLowerCase())){
+                    throw "文件类型不符合要求";
+                }
+                var tmp = {}
+                tmp["filename"] = fName
+                tmp["org_url"] = furl
+                if (fFile !== undefined) {
+                    fData.append("file", fFile, fName)
+                }
+                arr.push(tmp)
+            })
+        } catch(e){
+            showTip(e);
+            return;
+        }
+        if (arr.length > 0) {
+            fData.append("files", JSON.stringify(arr))
+        }
+        fData.append("id",{{.T.id}});
+        $('#uploadfile').modal('hide');
+        showLoading("正在上传附件，请稍候...")
+        $.ajax({
+            url: "/front/lua/uploadfile",
+            type: 'POST',
+            data: fData,
+            cache: false,
+            processData: false,
+            contentType: false,
+            success: function (r) {
+                hideLoading()
+                if (r&&r.success) {
+                        //window.location.reload();
+                        window.location.href = "/front/lua/warnedit?id="+{{.T.id}}+"&num=0";
+                } else {
+                    showTip(r.msg);
+                }
+            }
+        })
+    }
+</script>

src/web/templates/lua/warnlist.html

@@ -0,0 +1,246 @@
+{{include "com/inc.html"}}
+<!-- Main Header -->
+{{include "com/header.html"}}
+<!-- Left side column. 权限菜单 -->
+{{include "menu/menu.html"}}
+
+<!-- Content Wrapper. Contains page content -->
+<div class="content-wrapper">
+    <section class="content-header">
+        <h1>
+            <small>
+                <button type="button" class="btn btn-primary" data-toggle="modal" onclick="batchprocess(2)">批量删除</button>
+                <button type="button" class="btn btn-primary" data-toggle="modal" onclick="batchprocess(4)">批量发布</button>
+            </small>
+        </h1>
+        <ol class="breadcrumb">
+            <li><a href="#"><i class="fa fa-dashboard"></i> 爬虫数据维护</a></li>
+            <li><a href="/front/lua/warnlist"> 异常数据管理</a></li>
+        </ol>
+    </section>
+    <!-- Main content -->
+    <section class="content">
+        <div class="row">
+            <div class="col-xs-12">
+                <div class="box">
+                    <div class="box-body">
+                       <!-- <div>
+                            <div id="status-div" style="width: 205px;float: right">
+                                <span class="input-group date date-picker" id="starttime" data-provide="datepicker">
+                                    <input type="text" class="form-control form-filter input-sm" readonly name="starttime" placeholder="开始日期" />
+                                     <span class="input-group-addon">
+                                         <i class="fa fa-calendar"></i>
+                                      </span>
+                                </span>
+                            </div>
+                            <div style="display: flex;float: right;margin-right: 10px;align-items: center">
+                                <label style="white-space: nowrap;margin-bottom: 0;" for='name'>异常类型：</label>
+                                <select id='errtype' onchange='checkclick(this.value)' class='form-control input-sm'>
+                                    <option value='-1'>全部</option>
+                                    <option value='1'>警告</option>
+                                    <option value='2'>错误</option>
+                                </select>
+                            </div>
+                        </div>-->
+
+                        <table id="spiderwarn" class="table table-bordered table-hover">
+                            <thead>
+                            <tr>
+                                <th><input type="checkbox" id="selrow" onclick="selectrow(this)"/></th>
+                                <th>标题</th>
+                                <th>爬虫</th>
+                                <th>站点</th>
+                                <th>类型</th>
+                                <th>字段</th>
+                                <th>重复</th>
+                                <th>来源</th>
+                                <th>错误原因</th>
+                                <th>操作</th>
+                            </tr>
+                            </thead>
+                        </table>
+                    </div>
+                    <!-- /.box-body -->
+                </div>
+                <!-- /.box -->
+            </div>
+        </div>
+    </section>
+</div>
+
+<!-- footer -->
+{{include "com/footer.html"}}
+
+<script>
+    menuActive("warnlist");
+    $(function () {
+        ttable = $('#spiderwarn').DataTable({
+            "ajax": {
+                "url": "/front/lua/warnlist",
+                "type": "post",
+                "data": {}
+            },
+            "language": {
+                "url": "/dist/js/dataTables.chinese.lang"
+            },
+            "columnDefs": [
+                { "orderable": false, "targets": [0,1,3,6,7,8,9] } //设置列不可排序
+            ],
+            "order": [[2,"desc"]],
+            "paging": true,
+            "lengthChange": false,
+            "searching": true,
+            "ordering": true,
+            "info": true,
+            "serverSide": true,
+            "autoWidth":false,
+            "bLengthChange":true,//支持分页数选择
+            "columns": [
+                { "data": "_id",render:function(val,a,row){
+            				return "<input type='checkbox' value='"+val+"' />"
+            			}},
+                {
+                    "data": "title", render: function (val, a, row) {
+                        return val="<a href='"+row.href+"' title='"+row.title+"' target='_blank'>"+val+"</a>"
+                    }
+                },
+                {"data": "spidercode"},
+                {"data": "site"},
+                {"data": "level", render: function (val, a, row) {
+                    if(val == 2 ){
+                        return "错误";
+                    }
+                    return "警告";
+                }},
+                {"data": "field"},
+                {"data": "repeat",render: function(val){
+                    if(val){
+                        return "是"
+                    }else{
+                        return "否"
+                    }
+                    }},
+                {"data": "from"},
+                {"data": "info"},
+                {
+                  "data": "_id", render: function (val, a, row) {
+                      return "<a href='/front/lua/warnedit?id="+val+"&num=1'><i class='fa fa-fw fa-edit text-yellow'></i></a>";
+                  }
+                }
+            ],
+            "fnServerParams": function (e) {
+              if($('#starttime').length !=0){
+                var st = $('#starttime').datepicker('getDate')
+                if (st != null) {
+                  var s = st.toLocaleDateString()
+                  var sc = Date.parse(new Date(s).toString())/1000
+                  e.starttime = sc
+                }
+              }else{
+                var nst = Date.parse(new Date().toLocaleDateString())/1000;
+                e.starttime = nst
+              }
+              //repeat
+              var repeat = $("#repeat").val();
+              if(repeat){
+                  e.repeat = repeat;
+              }else{
+                  e.repeat= -1;
+              }
+              //来源
+                var from = $("#from").val();
+                if(from){
+                    e.from = from;
+                }else{
+                    e.from= -1;
+                }
+            }
+        });
+        ttable.on('init.dt', function () {
+          var errStype = "<div style='margin-top: -4px'>"+
+            "<span class='input-group date date-picker' id='starttime' data-provide='datepicker'>"+
+                "<input type='text' class='form-control form-filter input-sm' readonly name='starttime' placeholder='开始日期' />"+
+                 "<span class='input-group-addon'>"+
+                    "<i class='fa fa-calendar'></i>"+
+                  "</span>"+
+            "</span>"+
+         "</div>&nbsp;"
+          
+          var selectRepeat="<div class='form-group' style='margin-top: -4px'><label for='name'>是否重复：</label>"+
+              "<select id='repeat' onchange='checkclick(this.value)' class='form-control input-sm'>"+
+              "<option value='-1'>全部</option>"+
+              "<option value='1'>是</option>"+
+              "<option value='0'>否</option>"+
+              "</select></div>"
+          var selectFrom="<div class='form-group' style='margin-top: -4px'><label for='name'>来源：</label>"+
+              "<select id='from' onchange='checkclick(this.value)' class='form-control input-sm'>"+
+              "<option value='-1'>全部</option>"+
+              "<option value='list'>list</option>"+
+              "<option value='warn'>warn</option>"+
+              "</select></div>"
+          $("#spiderwarn_filter").prepend(errStype);
+          $("#spiderwarn_filter").prepend("&nbsp;");
+          $("#spiderwarn_filter").prepend(selectRepeat);
+          $("#spiderwarn_filter").prepend("&nbsp;&nbsp");
+          $("#spiderwarn_filter").prepend(selectFrom);
+          $("#spiderwarn_length").parent().css("width", "10%")
+          $("#spiderwarn_filter").parent().css("width", "90%")
+          $("#spiderwarn_filter").css({"display": "flex","text-align": "right","align-items": "center","justify-content": "flex-end"})
+
+          $('.date-picker').datepicker({
+              language: 'zh-CN',
+              autoclose: true,
+              clearBtn: true, //清除按钮
+              todayBtn: false, //今日按钮
+              format: "yyyy-mm-dd"
+          });
+          $('#starttime').datepicker('setDate',new Date().toLocaleDateString());//设置初始时间
+          $('#starttime').datepicker().on('changeDate', function (e) {
+              ttable.ajax.reload();
+          });
+        })
+    });
+    //批量删除
+    function batchprocess(state){
+      var ids=[];
+    		$("#spiderwarn td input[type=checkbox]").each(function(){
+    			if($(this).prop("checked")){
+    				ids.push($(this).attr("value"));
+    			}
+    		});
+      if(ids.length > 0){
+        showConfirm("确定批量处理？", function() {
+          $.ajax({
+            url: "/front/lua/spiderupdate",
+            type: "post",
+            data: {"ids":ids.join(","),"state":state},
+            success: function(r){
+              if(r&&r.success){
+                $("#selrow").prop('checked',false);
+                 ttable.ajax.reload();
+              }
+            }
+          })
+        })
+      }else{
+          if(state == 2){
+            alert("请选择要删除的数据！")
+          }else if(state == 4){
+              alert("请选择要发布的数据！")
+          }
+      }
+    }
+    
+    function checkclick(){
+        ttable.ajax.reload();
+    }
+    function selectrow(me){
+    		var sel=$(me);
+    		var isSelected=sel.prop('checked');
+    		if(isSelected){
+    			$("#spiderwarn td input[type=checkbox]").prop("checked",true);
+    		}else{
+    			$("#spiderwarn td input[type=checkbox]").prop("checked",false);
+    		}
+    	}
+</script>

src/web/templates/task/dataview.html

@@ -96,15 +96,13 @@
         var detail = document.getElementById('jsonDetail').value
         var tmp_detail = ''+detail
         if (tmp_detail=="") {
-            let msg = "请填写detail"
-            showMsg(msg)
+            showMsg("请填写detail")
             return;
         }
 
         var tmp_content = document.getElementById('jsonContentHtml').value
         if (tmp_content=="") {
-            let msg = "请填写contenthtml"
-            showMsg(msg)
+            showMsg("请填写contenthtml")
             return;
         }
 
@@ -173,7 +171,6 @@
     }
 
     function checkAddDict(data) {
-        return false
         if (data["title"]||data["site"]||data["spidercode"]||
             data["href"]||data["channel"]||data["toptype"]||
             data["subtype"]||data["area"]){