推送bidding信息支持修改

src/config.json

@@ -4,14 +4,14 @@
   "udport": 1483,
   "spiderchan": 5,
   "systemdb": {
-    "addr": "192.168.3.207:29099",
+    "addr": "192.168.3.71:29099",
     "db": "recapture",
     "size": 5,
     "username": "",
     "password": ""
   },
   "datadb": {
-    "addr": "192.168.3.207:29099",
+    "addr": "192.168.3.71:29099",
     "db": "recapture",
     "coll": "bidding",
     "size": 5,
@@ -19,21 +19,21 @@
     "password": ""
   },
   "luaspiderdb": {
-    "addr": "192.168.3.207:29099",
+    "addr": "192.168.3.71:29099",
     "db": "spider",
     "size": 5,
     "username": "",
     "password": ""
   },
   "bideditor": {
-    "addr": "192.168.3.207:29099",
+    "addr": "192.168.3.71:29099",
     "db": "editor",
     "size": 5,
     "username": "",
     "password": ""
   },
   "bidding": {
-    "addr": "192.168.3.207:29099",
+    "addr": "192.168.3.71:29099",
     "db": "qfw",
     "size": 5,
     "username": "",

src/front/lua.go

@@ -7,6 +7,7 @@ import (
 	"github.com/donnie4w/go-logger/logger"
 	"io/ioutil"
 	"mime/multipart"
+	"mongodb"
 	qu "qfw/util"
 	. "spiderutil"
 	"strings"
@@ -95,8 +96,9 @@ func (f *Front) WarnEdit() {
 	num, _ := f.GetInteger("num")
 	data := map[string]interface{}{}
 	bidId := ""
-	one, _ := MgoS.FindById("spider_warn_err", id, map[string]interface{}{"data": 1, "field": 1})
+	one, _ := MgoS.FindById("spider_warn_err", id, map[string]interface{}{"data": 1, "field": 1, "entry": 1})
 	if one != nil && len(*one) > 0 {
+		entry, _ := (*one)["entry"].(bool) //数据是否入库标记
 		data, _ = (*one)["data"].(map[string]interface{})
 		detail := qu.ObjToString((data)["detail"])
 		contenthtml := qu.ObjToString((data)["contenthtml"])
@@ -104,6 +106,7 @@ func (f *Front) WarnEdit() {
 		f.T["detail"] = detail
 		f.T["contenthtml"] = contenthtml
 		f.T["summary"] = summary
+		f.T["entry"] = entry
 		delete(data, "_id")
 		delete(data, "detail")
 		delete(data, "contenthtml")
@@ -202,128 +205,69 @@ func (f *Front) LuaDataSend() {
 			FormatFields(update)
 			FormatNumber(update) //解决超大金额转成科学计数法的问题
 		}
+		qu.Debug("update：", update)
 		one, _ := MgoS.FindById("spider_warn_err", id, map[string]interface{}{"data": 1})
 		if len(*one) > 0 {
 			data, _ := (*one)["data"].(map[string]interface{})
-			mustFiledNum := 0 //记录更新有效字段个数
-			result := map[string]interface{}{}
-			for k, _ := range Fields {
-				if v := update[k]; v != nil {
-					result[k] = v
-					mustFiledNum++
-				} else if v := data[k]; v != nil {
-					mustFiledNum++
-					result[k] = v
+			if stype == -1 { //bidding数据直接修改数据
+				success, msg = updateBidding(id, reasons, data, update)
+			} else { //非bidding数据推送保存服务
+				mustFiledNum := 0 //记录更新有效字段个数
+				result := map[string]interface{}{}
+				for k, _ := range Fields {
+					if v := update[k]; v != nil {
+						result[k] = v
+						mustFiledNum++
+					} else if v := data[k]; v != nil {
+						mustFiledNum++
+						result[k] = v
+					}
 				}
-			}
-			if mustFiledNum > 0 {
-				flag, bidid, bidcoll := SaveObj(4002, "title", result)
-				logger.Info(bid, "	", flag, "	", bidid, "	", bidcoll)
-				update["sendflag"] = flag
-				update["biddingid"] = bidid
-				update["biddingcoll"] = bidcoll
-				TagToSpiderWarnErr(stype, id, reasons, update) //spider_warn_err日志
-				if flag == "true" {
-					success = true
-					if bidcoll != "" && bidid != "" {
-						msg = "保存成功；数据所在表：" + bidcoll + "；数据ID：" + bidid
-					} else {
-						msg = "推送成功，但数据存在异常或被判重！"
+				if mustFiledNum > 0 {
+					flag, bidid, bidcoll := SaveObj(4002, "title", result)
+					logger.Info(bid, "	", flag, "	", bidid, "	", bidcoll)
+					update["sendflag"] = flag
+					update["biddingid"] = bidid
+					update["biddingcoll"] = bidcoll
+					TagToSpiderWarnErr(stype, id, reasons, update) //spider_warn_err日志
+					if flag == "true" {
+						success = true
+						if bidcoll != "" && bidid != "" {
+							msg = "保存成功；数据所在表：" + bidcoll + "；数据ID：" + bidid
+						} else {
+							msg = "推送成功，但数据存在异常或被判重！"
+						}
 					}
+				} else {
+					msg = "推送失败，无有效字段"
 				}
-			} else {
-				msg = "推送失败，无有效字段"
 			}
 		} else {
 			msg = "未找到该数据"
 		}
-		//if stype == 1 { //修复更新
-		//	if bid == "" {
-		//		rep = "bidding id 为空"
-		//		goto L
-		//	}
-		//	old_data, _ := JYMgo.FindById("bidding", bid, nil) //查询原始信息，用作修改日志记录
-		//	if old_data != nil && len(*old_data) > 0 {
-		//		delete(*old_data, "_id")
-		//		if (*old_data)["modifyinfo"] != nil {
-		//			if m, ok := (*old_data)["modifyinfo"].(map[string]interface{}); ok { //合并modifyinfo
-		//				for k, v := range m {
-		//					modifyinfo[k] = v
-		//				}
-		//			}
-		//		}
-		//	} else {
-		//		rep = "未找到bidding数据,id:" + bid
-		//		goto L
-		//	}
-		//	if len(modifyinfo) > 0 { //记录修改字段
-		//		update["modifyinfo"] = modifyinfo
-		//	}
-		//	b := JYMgo.UpdateById("bidding", bid, map[string]interface{}{"$set": update}) //更新
-		//	if b {                                                                        //更新成功                                                                   //更新成功 重生bidding索引 项目合并
-		//		qu.Debug("更新成功")
-		//		udptask.BiddingIndexUdp(bid, "bidding") //coll:bidding 是因为调用此方法的数据都是增量数据
-		//		if IsModifyPro(modifyinfo) {            //修改了影响项目合并的字段 进行项目合并
-		//			udptask.ProjectSetUdp(bid, "bidding")
-		//		}
-		//		delName1 := RedisDelKey1 + "*_" + bid
-		//		redis.DelByCodePattern(RedisJYName, delName1)
-		//		delete(update, "modifyinfo")
-		//		l.SaveUpdateLog(2, reasons, bid, *old_data, update, modifyinfo) //标签日志
-		//		TagToSpiderWarnErr(stype, id, reasons, update)                  //spider_warn_err日志
-		//		success = true
-		//	} else { //更新失败
-		//		rep = "更新bidding失败,id:" + bid
-		//		qu.Debug("更新失败")
-		//	}
-		//} else { //3:修复发布 4:直接发布
-		//	href := qu.ObjToString(update["href"])
-		//	shaid := qu.ObjToString(update["s_sha"])
-		//	competehref := qu.ObjToString(update["competehref"]) //竞品数据
-		//	iscompete, _ := update["iscompete"].(bool)           //新爬虫数据
-		//	newId := primitive.NewObjectID()
-		//	newSid := mongodb.BsonIdToSId(newId)
-		//	qu.Debug("newId:", newSid)
-		//	update["_id"] = newId
-		//	if competehref != "" { //竞品爬虫
-		//		href = competehref
-		//		update["href"] = GetJyHref(newSid)
-		//	} else if iscompete { //新爬虫
-		//		update["competehref"] = "#"
-		//	}
-		//	delete(update, "iscompete")
-		//	update["comeintime"] = time.Now().Unix()        //更新入库时间
-		//	update["modifyinfo"] = map[string]interface{}{} //添加空modifyinfo作为标识
-		//	PutHrefRedis(href, shaid, newSid)               //添加href进redis
-		//	qu.Debug(update)
-		//	JYMgo.SaveByOriID("bidding", update)                                                            //入bidding
-		//	l.SaveUpdateLog(1, reasons, newSid, map[string]interface{}{}, update, map[string]interface{}{}) //标签日志
-		//	TagToSpiderWarnErr(stype, id, reasons, update)                                                  //spider_warn_err日志
-		//	success = true
-		//}
 	}
 	f.ServeJson(map[string]interface{}{"success": success, "msg": msg})
 }
 
 func (f *Front) SpiderUpdate() {
 	defer qu.Catch()
-	state, _ := f.GetInteger("state")
+	stype, _ := f.GetInteger("stype")
 	reason := "批量处理无需更新"
-	if state == 4 { //批量直接发布
+	if stype == 4 { //批量直接发布
 		reason = "批量处理直接发布"
 	}
 	ids := strings.Split(f.GetString("ids"), ",")
 	for _, id := range ids {
-		TagToSpiderWarnErr(state, id, reason, map[string]interface{}{})
+		TagToSpiderWarnErr(stype, id, reason, map[string]interface{}{})
 	}
 	f.ServeJson(map[string]interface{}{"success": true})
 }
 
-func TagToSpiderWarnErr(state int, id, reason string, update map[string]interface{}) {
+func TagToSpiderWarnErr(stype int, id, reason string, update map[string]interface{}) {
 	defer qu.Catch()
 	set := map[string]interface{}{
 		"ok":         true,
-		"state":      state,
+		"stype":      stype,
 		"reason":     reason,
 		"updatetime": time.Now().Unix(),
 	}
@@ -440,3 +384,21 @@ func (f *Front) FilterSave() {
 //	}
 //	MgoS.Save(JyRecord, log_data)
 //}
+
+func updateBidding(errId, reasons string, data, update map[string]interface{}) (bool, string) {
+	title := qu.ObjToString(data["title"])
+	href := qu.ObjToString(data["href"])
+	spidercode := qu.ObjToString(data["spidercode"])
+	list, _ := MgoB.Find("bidding", map[string]interface{}{"title": title}, map[string]interface{}{"_id": -1}, map[string]interface{}{"title": 1, "href": 1, "spidercode": 1}, false, -1, -1)
+	for _, tmp := range *list {
+		hrefTmp := qu.ObjToString(tmp["href"])
+		spidercodeTmp := qu.ObjToString(tmp["spidercode"])
+		if hrefTmp == href && spidercodeTmp == spidercode {
+			if MgoB.UpdateById("bidding", tmp["_id"], map[string]interface{}{"$set": update}) {
+				TagToSpiderWarnErr(-1, errId, reasons, update) //spider_warn_err日志
+				return true, "修复成功；数据所在表：bidding；数据ID：" + mongodb.BsonIdToSId(tmp["_id"])
+			}
+		}
+	}
+	return false, "未匹配到bidding信息"
+}

src/web/templates/lua/filterlist.html

@@ -106,7 +106,7 @@
         });
     });
     //批量删除
-    function batchprocess(state){
+    function batchprocess(stype){
       var ids=[];
     		$("#filterdata td input[type=checkbox]").each(function(){
     			if($(this).prop("checked")){
@@ -118,7 +118,7 @@
           $.ajax({
             url: "/front/lua/spiderupdate",
             type: "post",
-            data: {"ids":ids.join(","),"state":state},
+            data: {"ids":ids.join(","),"stype":stype},
             success: function(r){
               if(r&&r.success){
                 $("#selrow").prop('checked',false);

src/web/templates/lua/warnedit.html

@@ -118,15 +118,19 @@
         <div class="nav-tabs-custom">
             <ul class="nav nav-tabs edit-step">
                 <button class="btn btn-primary btn-sm" style="float: left;margin-top: 7px;margin-left: 10px" onclick="showUploadFileModal()"><i class="fa fa-fw fa-file-text fa-lg"></i>补充附件</button>
-                {{if ne .T.bid ""}}
-                    <button class="btn btn-primary btn-sm" style="float: right;margin-top: 7px;margin-right: 10px" onclick="updateOrSave(1)">修复更新</button>
-                <button class="btn btn-primary btn-sm" style="float: right;margin-top: 7px;margin-right: 10px" onclick="updateOrSave(2)">无需更新</button>
+<!--                entry=true表示入bidding数据-->
+                {{if .T.entry}}
+                    <button class="btn btn-primary btn-sm" style="float: right;margin-top: 7px;margin-right: 10px" onclick="updateOrSave(-1)">修复Bidding</button>
                 {{else}}
+                    {{if ne .T.bid ""}}
+                    <button class="btn btn-primary btn-sm" style="float: right;margin-top: 7px;margin-right: 10px" onclick="updateOrSave(1)">修复更新</button>
+                    <button class="btn btn-primary btn-sm" style="float: right;margin-top: 7px;margin-right: 10px" onclick="updateOrSave(2)">无需更新</button>
+                    {{else}}
                     <button class="btn btn-primary btn-sm" style="float: right;margin-top: 7px;margin-right: 10px" onclick="updateOrSave(3)">修复发布</button>
-<!--                    <button class="btn btn-primary btn-sm" style="float: right;margin-top: 7px;margin-right: 10px" onclick="updateOrSave(4)">直接发布</button>-->
+                    <!--                    <button class="btn btn-primary btn-sm" style="float: right;margin-top: 7px;margin-right: 10px" onclick="updateOrSave(4)">直接发布</button>-->
                     <button class="btn btn-primary btn-sm" style="float: right;margin-top: 7px;margin-right: 10px" onclick="updateOrSave(2)">无需发布</button>
+                    {{end}}
                 {{end}}
-
             </ul>
 <!--            <div class="box-body">-->
 <!--                <div class="form-group">-->
@@ -178,18 +182,19 @@
     function updateOrSave(stype){
         /*
         * stype:
+        *   -1:bidding数据修复
         *   1:修复更新
         *   2:无需更新
         *   3:修复发布
         *   4:直接发布（现版本展示的都是错误数据，直接发布是无效的）
         * */
-         if (stype == 1 || stype == 3){
-            var reason=$("#reasons").val()
-            if (reason=="") {
-                showMsg("请填写更新数据原因")
-                return;
-            }
-        }
+        //  if (stype == -1 || stype == 1 || stype == 3){
+        //     var reason=$("#reasons").val()
+        //     if (reason=="") {
+        //         showMsg("请填写更新数据原因")
+        //         return;
+        //     }
+        // }
 
         var data = JSON.parse($("#jsonTextarea").val())
         // 字段非空校验
@@ -209,10 +214,13 @@
             showMsg("请填写contenthtml")
             return;
         }
-
         // 判断除detail、contenthtml、summary外字段是否修改
         var modifyinfo = []
         var update = {}
+        if (stype == -1){
+            modifyinfo.push("projectinfo")
+            update["projectinfo"] = data["projectinfo"]
+        }
         // 删除、修改
         for (let editDataKey in edit_data) {
             if (typeof edit_data[editDataKey] == "object") {
@@ -268,7 +276,7 @@
                     "update": JSON.stringify(update),
                     "bid": bid,
                     "id": {{.T.id}},
-                    "reasons": reason,
+                    "reasons": "",
                     "stype": stype
                 },
                 success: function (r) {
@@ -311,8 +319,11 @@
         var fData = new FormData();
         try {
             $('.group-item').each(function () {
-                var furl = $(this).find("input.furl").val()
-                var fFile = $(this).find('input.ffile')[0].files[0]
+                var furl = $(this).find("input.furl").val();
+                if(furl == ""){
+                    throw "附件地址不能不空！";
+                }
+                var fFile = $(this).find('input.ffile')[0].files[0];
                 if (fFile === undefined) {
                     throw "请选择本地文件";
                 }

src/web/templates/lua/warnlist.html

@@ -214,12 +214,12 @@
                 {"data": "from"},
                 {"data": "info"},
                 {"data": "_id", render: function (val, a, row) {
-                    if(row.entry){
-                        //已入库的异常数据目前修复是删除redis再入库一条，会被判重，故当前不允许编辑，只能通过导入任务修复
-                        return "<a  href='JavaScript:void(0)'><i class='fa fa-fw fa-edit text-yellow'></i></a>";
-                    }else{
-                        return "<a href='/front/lua/warnedit?id="+val+"&num=1' target='_blank'><i class='fa fa-fw fa-edit text-yellow'></i></a>";
-                    }
+                    return "<a href='/front/lua/warnedit?id="+val+"&num=1' target='_blank'><i class='fa fa-fw fa-edit text-yellow'></i></a>";
+                    // if(row.entry){
+                    //     return "<a  href='JavaScript:void(0)'><i class='fa fa-fw fa-edit text-yellow'></i></a>";
+                    // }else{
+                    //     return "<a href='/front/lua/warnedit?id="+val+"&num=1' target='_blank'><i class='fa fa-fw fa-edit text-yellow'></i></a>";
+                    // }
                 }}
             ],
             "fnServerParams": function (e) {
@@ -325,7 +325,7 @@
         })
     });
     //批量删除
-    function batchprocess(state){
+    function batchprocess(stype){
       var ids=[];
     		$("#spiderwarn td input[type=checkbox]").each(function(){
     			if($(this).prop("checked")){
@@ -337,7 +337,7 @@
           $.ajax({
             url: "/front/lua/spiderupdate",
             type: "post",
-            data: {"ids":ids.join(","),"state":state},
+            data: {"ids":ids.join(","),"stype":stype},
             success: function(r){
               if(r&&r.success){
                 $("#selrow").prop('checked',false);