bidding数据导入及任务相关调整

src/config.json

@@ -4,14 +4,14 @@
   "udport": 1483,
   "spiderchan": 5,
   "systemdb": {
-    "addr": "192.168.3.207:27092",
+    "addr": "192.168.3.207:29099",
     "db": "recapture",
     "size": 5,
     "username": "",
     "password": ""
   },
   "datadb": {
-    "addr": "192.168.3.207:27092",
+    "addr": "192.168.3.207:29099",
     "db": "recapture",
     "coll": "bidding",
     "size": 5,
@@ -19,22 +19,22 @@
     "password": ""
   },
   "luaspiderdb": {
-    "addr": "192.168.3.207:27092",
+    "addr": "192.168.3.207:29099",
     "db": "spider",
     "size": 5,
     "username": "",
     "password": ""
   },
   "bideditor": {
-    "addr": "192.168.3.207:27092",
+    "addr": "192.168.3.207:29099",
     "db": "editor",
     "size": 5,
     "username": "",
     "password": ""
   },
   "bidding": {
-    "addr": "192.168.3.207:27092",
-    "db": "editor",
+    "addr": "192.168.3.207:29099",
+    "db": "qfw",
     "size": 5,
     "username": "",
     "password": ""
@@ -62,7 +62,7 @@
   "jsserveraddress":  "127.0.0.1:8031",
   "redisclusteraddrs": [
     "192.168.3.207:2179",
-    "192.168.3.166:2379"
+    "192.168.3.207:2279"
   ],
   "word":{
     "keyword":"(抽签|中标|招标|成交|合同|中标候选人|资格预审|拟建|邀请|询价|比选|议价|竞价|磋商|采购|招投标|答疑|变更公告|更正公告|竞争性谈判|竞谈|意见征询|澄清|单一来源|流标|废标|验收公告|中止|终止|违规|处罚|征集公告|开标结果|评审结果|监理|招租|租赁|评判结果|项目|遴选|补遗|竞标|征求意见)",

src/front/data.go

@@ -9,6 +9,7 @@ import (
 	qu "qfw/util"
 	"regexp"
 	"strings"
+	"sync"
 	. "task"
 	. "udptask"
 	"util"
@@ -31,6 +32,116 @@ var fileIndexMap = map[string]bool{
 var publishtimeReg = regexp.MustCompile("\\d{4}-\\d{2}-\\d{2}")
 var fields = map[string]interface{}{"state": 1, "spidercode": 1, "site": 1, "channel": 1, "title": 1, "href": 1}
 
+func (f *Front) PrepareBidding() {
+	defer qu.Catch()
+	sess := util.MgoDT.GetMgoConn()
+	defer util.MgoDT.DestoryMongoConn(sess)
+	lock := &sync.Mutex{}
+	wg := &sync.WaitGroup{}
+	ch := make(chan bool, 10)
+	fields := map[string]interface{}{
+		"spidercode": 1,
+		"title":      1,
+		"href":       1,
+	}
+	query := map[string]interface{}{
+		"state": map[string]interface{}{
+			"$ne": 1,
+		},
+	}
+	count := util.MgoDT.Count("bidding_copy", query)
+	it := sess.DB(util.MgoDT.DbName).C("bidding_copy").Find(&query).Select(&fields).Iter()
+	n := 0
+	oknum := 0
+	arr := [][]map[string]interface{}{}
+	save := []map[string]interface{}{}
+	for tmp := make(map[string]interface{}); it.Next(tmp); n++ {
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			list, _ := util.MgoB.Find("bidding", map[string]interface{}{"title": tmp["title"]}, nil, nil, false, -1, -1)
+			state := 0 //标记是否找到对应的bidding数据
+			for _, l := range *list {
+				if qu.ObjToString(l["href"]) == qu.ObjToString(tmp["href"]) && qu.ObjToString(l["spidercode"]) == qu.ObjToString(tmp["spidercode"]) {
+					state = 1
+					lock.Lock()
+					oknum++
+					l["state"] = 0
+					save = append(save, l)
+					lock.Unlock()
+					break
+				}
+			}
+			update := []map[string]interface{}{}
+			update = append(update, map[string]interface{}{"_id": tmp["_id"]})
+			update = append(update, map[string]interface{}{"$set": map[string]interface{}{"state": state}})
+			lock.Lock()
+			arr = append(arr, update)
+			if len(arr) > 500 {
+				util.MgoDT.UpdateBulk("bidding_copy", arr...)
+				arr = [][]map[string]interface{}{}
+			}
+			if len(save) > 500 {
+				util.MgoDT.SaveBulk(util.DataColl, save...)
+				save = []map[string]interface{}{}
+			}
+			lock.Unlock()
+		}(tmp)
+		if n%1000 == 0 {
+			qu.Debug("current:", n)
+		}
+		tmp = map[string]interface{}{}
+	}
+	wg.Wait()
+	if len(arr) > 0 {
+		util.MgoDT.UpdateBulk("bidding_copy", arr...)
+		arr = [][]map[string]interface{}{}
+	}
+	if len(save) > 0 {
+		util.MgoDT.SaveBulk(util.DataColl, save...)
+		save = []map[string]interface{}{}
+	}
+	msg := "bidding_copy表共" + fmt.Sprint(count) + "条数据，成功拉取对应bidding信息" + fmt.Sprint(oknum) + "条"
+	qu.Debug("bidding表获取信息结果：", msg)
+	f.ServeJson(map[string]interface{}{"ok": true, "msg": msg})
+}
+
+func (f *Front) ViewBidding() {
+	defer qu.Catch()
+	if f.Method() == "POST" {
+		searchStr := f.GetString("search[value]")
+		search := strings.TrimSpace(searchStr)
+		start, _ := f.GetInteger("start")
+		limit, _ := f.GetInteger("length")
+		draw, _ := f.GetInteger("draw")
+		query := map[string]interface{}{}
+		state, _ := f.GetInteger("state")
+		if state != -1 {
+			query["state"] = state
+		}
+		if search != "" {
+			query["$or"] = []interface{}{
+				map[string]interface{}{"title": map[string]interface{}{"$regex": search}},
+			}
+		}
+		qu.Debug("query:", query)
+		task, _ := util.MgoDT.Find("bidding_copy", query, map[string]interface{}{"_id": -1}, nil, false, start, limit)
+		count := util.MgoDT.Count("bidding_copy", query)
+		f.ServeJson(map[string]interface{}{
+			"draw":            draw,
+			"data":            *task,
+			"recordsFiltered": count,
+			"recordsTotal":    count,
+		})
+	} else {
+		f.Render("task/biddingview.html")
+	}
+}
+
 func (f *Front) ImportData() {
 	defer qu.Catch()
 	dataFile, _, err := f.GetFile("xlsx")
@@ -75,7 +186,7 @@ func (f *Front) ImportData() {
 								result[field] = publishtime
 							} else if cell.NumFmt == "m/d/yy h:mm" {
 								t, _ := cell.GetTime(false)
-								result[field] = qu.FormatDate(&t, qu.Date_Full_Layout)
+								result[field] = t.Format(qu.Date_Full_Layout)
 							} else { //其他类型视为异常
 								ok = false
 								msgArr = append(msgArr, "第"+fmt.Sprint(i+1)+"行publishtime字段异常")
@@ -279,6 +390,29 @@ func (f *Front) DataUpdate() {
 	f.ServeJson(map[string]interface{}{"rep": ok, "msg": msg})
 }
 
+func (f *Front) DataDelete() {
+	defer qu.Catch()
+	taskid := f.GetString("taskid")
+	task, _ := util.Mgo.FindById("task", taskid, nil)
+	ok := false
+	msg := ""
+	if len(*task) > 0 {
+		coll := qu.ObjToString((*task)["s_coll"])
+		code := f.GetString("code")
+		codeArr := strings.Split(code, ",")
+		query := map[string]interface{}{
+			"spidercode": map[string]interface{}{
+				"$in": codeArr,
+			},
+		}
+		qu.Debug("删除无效数据的爬虫：", code)
+		ok = util.MgoDT.Del(coll, query)
+	} else {
+		msg = "任务信息查询失败"
+	}
+	f.ServeJson(map[string]interface{}{"msg": msg, "ok": ok})
+}
+
 func checkField(result map[string]interface{}) (arr []string) {
 	defer qu.Catch()
 	for _, f := range []string{"href", "title", "site", "channel", "area", "spidercode", "publishtime"} {

src/front/front.go

@@ -52,9 +52,12 @@ type Front struct {
 	dataView   xweb.Mapper `xweb:"/front/task/dataview"`   //数据详情
 	dataSend   xweb.Mapper `xweb:"/front/task/datasend"`   //数据推送
 	dataUpdate xweb.Mapper `xweb:"/front/task/dataupdate"` //数据更新推送
+	dataDelete xweb.Mapper `xweb:"/front/task/datadelete"` //数据更新推送
 
 	//data
-	importData xweb.Mapper `xweb:"/front/data/importdata"` //数据导入
+	importData     xweb.Mapper `xweb:"/front/data/importdata"`     //数据导入
+	prepareBidding xweb.Mapper `xweb:"/front/data/preparebidding"` //基于基本信息，检索bidding表对应数据
+	viewBidding    xweb.Mapper `xweb:"/front/data/viewbidding"`    //展示拉取bidding的结果
 
 	//lua
 	warnList     xweb.Mapper `xweb:"/front/lua/warnlist"`     //spider_warn异常数据

src/front/task.go

@@ -66,6 +66,7 @@ func (f *Front) TaskSave() {
 		user := f.GetSession("user").(map[string]interface{})
 		id := f.GetString("id")
 		name := f.GetString("name")
+		checkfields := f.GetString("checkfields")
 		isbidding := f.GetString("isbidding")
 		flows := f.GetString("flows")
 		flowsArr := []string{}
@@ -80,6 +81,7 @@ func (f *Front) TaskSave() {
 		set := map[string]interface{}{}
 		set["s_name"] = name
 		set["s_coll"] = coll
+		set["s_checkfields"] = checkfields
 		set["isbidding"] = isbidding == "是"
 		set["flows"] = flowsArr
 		if id != "" {
@@ -116,43 +118,94 @@ func (f *Front) TaskStart() {
 	task, _ := Mgo.FindById("task", id, nil)
 	qu.Debug("启动任务：", (*task)["s_name"], id)
 	msg := ""
+	msgType := 0
+	errCode := []string{}
 	if len(*task) > 0 {
 		coll := qu.ObjToString((*task)["s_coll"])
-		TaskLock.Lock()
-		if RunningTask[coll] {
-			msg = "表：" + coll + "已有任务正在运行，请稍后启动该任务"
-		} else { //启动任务
-			//更新数据状态，设置默认字段state=0,times=0
-			query := map[string]interface{}{
-				"$or": []interface{}{
-					map[string]interface{}{"state": map[string]interface{}{"$exists": false}}, //首次执行任务检索
-					map[string]interface{}{"state": -1}},                                      //任务重启检索
-			}
-			MgoDT.Update(coll, query, map[string]interface{}{"$set": map[string]interface{}{"state": 0, "times": 0}}, false, true)
-			if MgoDT.Count(coll, map[string]interface{}{"state": 0}) > 0 {
-				//更新任务状态
-				Mgo.UpdateById("task", id, map[string]interface{}{"$set": map[string]interface{}{"i_state": 1, "issend": false, "l_starttime": time.Now().Unix()}})
-				RunningTask[coll] = true
-				t := &Task{
-					ID:              mongodb.BsonIdToSId((*task)["_id"]),
-					Name:            qu.ObjToString((*task)["s_name"]),
-					DB:              qu.ObjToString((*task)["s_db"]),
-					Coll:            qu.ObjToString((*task)["s_coll"]),
-					IsBidding:       (*task)["isbidding"].(bool),
-					Flows:           qu.ObjArrToStringArr((*task)["flows"].([]interface{})),
-					SpiderScriptMap: map[string]*spider.Spider{},
+		//校验coll中数据对应爬虫是否已弃用
+		errCode = CheckCodeState(coll)
+		if len(errCode) == 0 {
+			TaskLock.Lock()
+			if RunningTask[coll] {
+				msg = "表：" + coll + "已有任务正在运行，请稍后启动该任务"
+			} else { //启动任务
+				//更新数据状态，设置默认字段state=0,times=0
+				query := map[string]interface{}{
+					"$or": []interface{}{
+						map[string]interface{}{"state": map[string]interface{}{"$exists": false}}, //首次执行任务检索
+						map[string]interface{}{"state": -1}},                                      //任务重启检索
+				}
+				MgoDT.Update(coll, query, map[string]interface{}{"$set": map[string]interface{}{"state": 0, "times": 0}}, false, true)
+				if MgoDT.Count(coll, map[string]interface{}{"state": 0}) > 0 {
+					//更新任务状态
+					Mgo.UpdateById("task", id, map[string]interface{}{"$set": map[string]interface{}{"i_state": 1, "issend": false, "l_starttime": time.Now().Unix()}})
+					RunningTask[coll] = true
+					t := &Task{
+						ID:              mongodb.BsonIdToSId((*task)["_id"]),
+						Name:            qu.ObjToString((*task)["s_name"]),
+						DB:              qu.ObjToString((*task)["s_db"]),
+						Coll:            qu.ObjToString((*task)["s_coll"]),
+						IsBidding:       (*task)["isbidding"].(bool),
+						Flows:           qu.ObjArrToStringArr((*task)["flows"].([]interface{})),
+						SpiderScriptMap: map[string]*spider.Spider{},
+					}
+					fieldsMap := map[string]bool{}
+					if checkFields, ok := (*task)["s_checkfields"].(string); ok && checkFields != "" {
+						if fieldsArr := strings.Split(checkFields, ","); len(fieldsArr) >= 1 {
+							for _, field := range fieldsArr {
+								fieldsMap[field] = true
+							}
+						}
+					}
+					t.CheckFields = fieldsMap
+					go t.StartTask() //启动任务
+				} else {
+					msg = "表：" + coll + "无有效数据"
 				}
-				go t.StartTask() //启动任务
-			} else {
-				msg = "表：" + coll + "无有效数据"
 			}
+			TaskLock.Unlock()
+		} else {
+			msg = "启动任务失败，重采数据中：" + strings.Join(errCode, ",") + "爬虫已失效，是否删除对应数据！"
+			msgType = 1
 		}
-		TaskLock.Unlock()
 	} else {
 		msg = "启动任务失败，未找到该任务"
 	}
 	if msg != "" {
 		logger.Info("任务启动失败：", (*task)["s_name"], id)
 	}
-	f.ServeJson(map[string]interface{}{"msg": msg})
+	f.ServeJson(map[string]interface{}{"msg": msg, "msgtype": msgType, "errcode": strings.Join(errCode, ",")})
+}
+
+func CheckCodeState(coll string) (errCode []string) {
+	defer qu.Catch()
+	match := map[string]interface{}{
+		"state": map[string]interface{}{
+			"$ne": 1,
+		},
+	}
+	group := map[string]interface{}{
+		"_id": "$spidercode",
+	}
+	p := []map[string]interface{}{
+		map[string]interface{}{"$match": match},
+		map[string]interface{}{"$group": group},
+	}
+	sess := MgoDT.GetMgoConn()
+	defer MgoDT.DestoryMongoConn(sess)
+	it2 := sess.DB(MgoDT.DbName).C(coll).Pipe(p).Iter()
+	n2 := 0
+	for tmp := make(map[string]interface{}); it2.Next(&tmp); n2++ {
+		code := qu.ObjToString(tmp["_id"]) //爬虫代码
+		q := map[string]interface{}{
+			"code":     code,
+			"state":    5,
+			"platform": "golua平台",
+		}
+		if MgoE.Count("luaconfig", q) == 0 {
+			errCode = append(errCode, code)
+		}
+		tmp = map[string]interface{}{}
+	}
+	return
 }

src/task/task.go

@@ -27,6 +27,7 @@ type Task struct {
 	IsBidding       bool
 	Flows           []string
 	SpiderScriptMap map[string]*spider.Spider
+	CheckFields     map[string]bool
 }
 
 func (t *Task) StartTask() {
@@ -116,9 +117,10 @@ func (t *Task) StartJob() {
 			//每个爬虫加载需要下载的数据
 			q := map[string]interface{}{ //查询未下载成功的数据
 				"spidercode": spidercode,
-				"state": map[string]interface{}{
-					"$ne": 1,
-				},
+				"state":      0,
+				//"state": map[string]interface{}{
+				//	"$ne": 1,
+				//},
 			}
 			fields := map[string]interface{}{
 				"href":        1,
@@ -131,36 +133,36 @@ func (t *Task) StartJob() {
 				list, _ := MgoDT.Find(t.Coll, q, nil, fields, false, 0, 100)
 				if list != nil && len(*list) > 0 {
 					updateArr := [][]map[string]interface{}{}
-					for _, tmp := range *list {
+					for _, l := range *list {
 						//_id := tmp["_id"]
 						update := []map[string]interface{}{}
 						data := map[string]interface{}{}
-						result := map[string]interface{}{}
-						query := map[string]interface{}{"_id": tmp["_id"]}
-						times := qu.IntAll(tmp["times"]) //获取下载次数
+						tmp := map[string]interface{}{}
+						query := map[string]interface{}{"_id": l["_id"]}
+						times := qu.IntAll(l["times"]) //获取下载次数
 						//publishtime处理成字符串
-						if publishtime, ok := tmp["publishtime"].(int64); ok { //int64
+						if publishtime, ok := l["publishtime"].(int64); ok { //int64
 							data["publishtime"] = qu.FormatDateByInt64(&publishtime, qu.Date_Full_Layout)
-						} else if publishtime, ok := tmp["publishtime"].(string); ok {
+						} else if publishtime, ok := l["publishtime"].(string); ok {
 							data["publishtime"] = publishtime
 						}
 						//jsondata处理成字符串
-						if jd, ok := tmp["jsondata"].(map[string]interface{}); ok && jd != nil {
+						if jd, ok := l["jsondata"].(map[string]interface{}); ok && jd != nil {
 							if jd_byte, err := json.Marshal(jd); err == nil && string(jd_byte) != "" {
 								data["jsondata"] = string(jd_byte)
 							}
-						} else if jd, ok := tmp["jsondata"].(string); ok && jd != "" {
+						} else if jd, ok := l["jsondata"].(string); ok && jd != "" {
 							data["jsondata"] = jd
 						}
-						data["title"] = tmp["title"]
-						data["href"] = tmp["href"]
+						data["title"] = l["title"]
+						data["href"] = l["href"]
 						var err interface{}
 						//下载详情页
-						result, err = sp.DownloadDetailPage(data, result)
+						tmp, err = sp.DownloadDetailPage(data, tmp)
 						//删除多余字段
 						//delete(result, "exit")
 						//delete(result, "checkpublishtime")
-						if err != nil || len(result) == 0 { //下载失败
+						if err != nil || len(tmp) == 0 { //下载失败
 							times++
 							ss := map[string]interface{}{"times": times}
 							if times >= 3 { //3次下载失败今天不再下载,state置为1
@@ -173,7 +175,7 @@ func (t *Task) StartJob() {
 							continue
 						}
 						//正文、附件分析,下载异常数据重新下载
-						if AnalysisProjectInfo(result) {
+						if AnalysisProjectInfo(tmp) {
 							times++
 							ss := map[string]interface{}{"times": times}
 							if times >= 3 { //3次下载失败今天不再下载,state置为-1
@@ -185,18 +187,19 @@ func (t *Task) StartJob() {
 							updateArr = append(updateArr, update)
 							continue
 						}
-						l_np_publishtime := qu.Int64All(result["l_np_publishtime"])
+						l_np_publishtime := qu.Int64All(tmp["l_np_publishtime"])
 						if l_np_publishtime > time.Now().Unix() || l_np_publishtime == 0 { //防止发布时间超前
 							l_np_publishtime = time.Now().Unix()
 						}
-						delete(result, "l_np_publishtime")
-						result["publishtime"] = l_np_publishtime
+						delete(tmp, "l_np_publishtime")
+						tmp["publishtime"] = l_np_publishtime
 						if !t.IsBidding { //不是bidding数据，走保存服务需要以下字段
-							result["iscompete"] = sp.IsCompete //2021-11-01以后新增的爬虫不在展示原文链接(保存服务判断)
+							tmp["iscompete"] = sp.IsCompete //2021-11-01以后新增的爬虫不在展示原文链接(保存服务判断)
 							//spider.Store(sp.StoreMode, sp.StoreToMsgEvent, sp.Collection, sp.CoverAttr, result, true)
 						}
+						tmp["state"] = 1
+						result := t.CheckField(tmp) //校验字段
 						//下载成功
-						result["state"] = 1
 						update = append(update, query)
 						update = append(update, map[string]interface{}{"$set": result})
 						updateArr = append(updateArr, update)
@@ -217,6 +220,51 @@ func (t *Task) StartJob() {
 	logger.Info(t.Name, t.ID, t.Coll, "重采完毕...")
 }
 
+//字段校验
+func (t *Task) CheckField(tmp map[string]interface{}) map[string]interface{} {
+	defer qu.Catch()
+	result := map[string]interface{}{}
+	if len(t.CheckFields) > 0 {
+		for field, _ := range t.CheckFields {
+			fieldOk := true
+			if field == "projectinfo" { //附件信息校验
+				if projectinfo, ok := tmp["projectinfo"].(map[string]interface{}); ok && len(projectinfo) > 0 {
+					if attachments, ok := projectinfo["attachments"].(map[string]interface{}); ok && len(attachments) > 0 {
+						for _, data := range attachments {
+							if d, ok := data.(map[string]interface{}); ok {
+								fid := qu.ObjToString(d["fid"])
+								if fid == "" { //附件上传成功
+									fieldOk = false
+									break
+								}
+							}
+						}
+					} else {
+						fieldOk = false
+					}
+				} else {
+					fieldOk = false
+				}
+			} else { //其它字段校验
+				if tmp[field] == nil || qu.ObjToString(tmp[field]) == "" {
+					fieldOk = false
+				}
+			}
+			if !fieldOk { //字段值下载出错，该条数据下载失败
+				result["state"] = -1
+				return result
+			} else if fieldOk && t.IsBidding { //bidding数据指定更新某字段
+				result[field] = tmp[field]
+			}
+		}
+		if t.IsBidding {
+			result["state"] = 1
+			return result
+		}
+	}
+	return tmp
+}
+
 //数据推送
 func (t *Task) SendData() {
 	if t.IsBidding { //1、bidding数据推送

src/util/config.go

@@ -10,7 +10,7 @@ import (
 )
 
 var (
-	Mgo      *mongodb.MongodbSim
+	Mgo      *mongodb.MongodbSim //程序基本库
 	MgoDT    *mongodb.MongodbSim //数据存储库
 	MgoE     *mongodb.MongodbSim //爬虫库
 	MgoB     *mongodb.MongodbSim //bidding库
@@ -34,8 +34,8 @@ func InitMgoPool() {
 		MongodbAddr: Config.SystemDB.Addr,
 		Size:        Config.SystemDB.Size,
 		DbName:      Config.SystemDB.Db,
-		//UserName:    qu.ObjToString(Sysconfig["uname"]),
-		//Password:    qu.ObjToString(Sysconfig["upwd"]),
+		UserName:    Config.SystemDB.Username,
+		Password:    Config.SystemDB.Password,
 	}
 	Mgo.InitPool()
 	//重采数据库

src/web/templates/task/biddingview.html

@@ -0,0 +1,105 @@
+{{include "com/inc.html"}}
+<!-- Main Header -->
+{{include "com/header.html"}}
+<!-- Left side column. 权限菜单 -->
+{{include "menu/menu.html"}}
+<!-- Content Wrapper. Contains page content -->
+<div class="content-wrapper">
+    <section class="content-header">
+        <h1>
+            <small>
+                <iframe srcdoc="
+                    <form id='uploadform' method='post' enctype='multipart/form-data' action='/front/data/importdata'>
+                        <input type='file' name='xlsx' />
+                    </form>" height=0 scrolling=no class="hide"  id="fileframe">
+                </iframe>
+            </small>
+        </h1>
+        <ol class="breadcrumb">
+            <li><a href="/front/task"><i class="fa fa-dashboard"></i> 任务列表</a></li>
+        </ol>
+    </section>
+    <!-- Main content -->
+    <section class="content">
+        <div class="row">
+            <div class="col-xs-12">
+                <div class="box">
+                    <div class="box-body">
+                        <table id="bidding" class="table table-bordered table-hover">
+                            <thead>
+                            <tr>
+                                <th>标题</th>
+                                <th>链接</th>
+                                <th>数据拉取</th>
+                            </tr>
+                            </thead>
+                        </table>
+                    </div>
+                    <!-- /.box-body -->
+                </div>
+                <!-- /.box -->
+            </div>
+        </div>
+    </section>
+</div>
+
+<!-- footer -->
+{{include "com/footer.html"}}
+
+<script>
+    menuActive("bidding");
+    $(function () {
+        ttable = $('#bidding').DataTable({
+            "paging": true,
+            "lengthChange": false,
+            "searching": true,
+            "ordering": false,
+            "info": true,
+            "autoWidth": true,
+            "serverSide": true,
+            "ajax": {
+                "url": "/front/data/viewbidding",
+                "type": "post",
+                "data": {}
+            },
+            "language": {
+                "url": "/dist/js/dataTables.chinese.lang"
+            },
+            "columns": [
+                {"data": "title"},
+                {"data": "href"},
+                {"data": "_id",render:function (v,a,r) {
+                    if (r.state == 1){
+                        return "成功";
+                    }else{
+                        return "失败";
+                    }
+                }},
+            ],
+            "fnServerParams": function (e) {
+                var state=$("#state").val();
+                if(state){
+                    e.state=state;
+                }else{
+                    e.state="-1";
+                }
+            }
+        });
+        ttable.on('init.dt', function () {
+            var state="<option value='-1'>全部</option>"+
+                "<option value='0'>失败</option>"+
+                "<option value='1'>成功</option>";
+            var selectbok="<div class='form-group'><label for='name'>数据拉取：</label>"+
+                "<select id='state' onchange='checkclick(this.value)' class='form-control input-sm'>"+
+                state+
+                "</select></div>"
+            $("#bidding_filter").prepend("&nbsp;&nbsp;");
+            $("#bidding_filter").prepend(selectbok);
+            $("#bidding_wrapper .col-sm-6").css({width:"100%"});
+        });
+    })
+
+    function checkclick(){
+        ttable.ajax.reload();
+    }
+</script>

src/web/templates/task/task.html

@@ -34,8 +34,10 @@
     <section class="content-header">
         <h1>
             <small>
-                <button type="button" class="btn btn-primary" data-toggle="modal" onclick="add()">新建任务</button>
-                <button type="button" class="btn btn-primary" data-toggle="modal" onclick="importdata()">导入数据</button>
+                <button type="button" class="btn btn-success" data-toggle="modal" onclick="add()">新建任务</button>
+                <button type="button" class="btn btn-warning" data-toggle="modal" onclick="importdata()">导入数据</button>
+                <button type="button" class="btn btn-primary" data-toggle="modal" onclick="preparebidding()">准备bidding</button>
+                <button type="button" class="btn btn-primary" data-toggle="modal" onclick="window.open('/front/data/viewbidding/')">bidding信息</button>
                 <iframe srcdoc="
                     <form id='uploadform' method='post' enctype='multipart/form-data' action='/front/data/importdata'>
                         <input type='file' name='xlsx' />
@@ -107,6 +109,12 @@
                             <input id="coll" name="coll" type="text" class="form-control" value="{{.T.datacoll}}">
                         </div>
                     </div>
+                    <div class="form-group">
+                        <label for="checkfields" class="col-sm-2 control-label">校验字段:</label>
+                        <div class="col-sm-10">
+                            <input id="checkfields" name="checkfields" type="text" class="form-control" value="{{.T.checkfields}}"  placeholder="请输入重采校验字段 例：title,detail(多字段逗号分割)">
+                        </div>
+                    </div>
                     <div class="form-group">
                         <label for="isbidding" class="col-sm-2 control-label">Bid数据:</label>
                         <div class="col-sm-10">
@@ -146,7 +154,6 @@
     <!-- /.modal-dialog -->
 </div>
 <!-- /.modal -->
-
 <!-- footer -->
 {{include "com/footer.html"}}
 
@@ -207,7 +214,7 @@
                     }},
                 {
                     "data": "_id", render: function (val, a, row) {
-                        return "<a href='#' onclick='edit(\"" + val + "\",\"" + row['s_name'] + "\",\"" + row['s_db'] + "\",\"" + row['s_coll'] + "\",\"" + row['isbidding'] + "\",\"" + row['flows'] + "\")'><i class='fa fa-fw fa-edit text-yellow'></i></a> &nbsp;" +
+                        return "<a href='#' onclick='edit(\"" + val + "\",\"" + row['s_name'] + "\",\"" + row['s_db'] + "\",\"" + row['s_coll'] + "\",\"" + row['s_checkfields'] + "\",\"" + row['isbidding'] + "\",\"" + row['flows'] + "\")'><i class='fa fa-fw fa-edit text-yellow'></i></a> &nbsp;" +
                             "<a href='#' onclick='del(\"" + val + "\")'><i class='fa fa-fw fa-trash text-red'></i></a>"
                     }
                 },
@@ -215,7 +222,7 @@
                     "data": "_id",width:"60px", render: function (val, a, row) {
                         var result = "<a href='/front/task/datalist?coll="+row['s_coll']+"&taskid="+row['_id']+"' class='btn btn-sm btn-primary'>结果查询</a>"
                         if (!row.issend && row.i_state == 2){
-                            result  = result +"<a href='#'onclick='datasend(\"" + row['_id'] + "\")' class='btn btn-sm btn-primary'>数据推送</a>"
+                            result  = result +"<a href='#' onclick='datasend(\"" + row['_id'] + "\")' class='btn btn-sm btn-success'>数据推送</a>"
                         }
                         return result
 
@@ -258,12 +265,36 @@
                         showTip("启动成功", 2000, function () { });
                         ttable.ajax.reload();
                     } else {
-                        showTip(r.msg, 3000, function () {});
+                        if(r.msgtype ==0){
+                            showTip(r.msg, 3000, function () { })
+                        }else if (r.msgtype ==1){
+                            showConfirm(r.msg, function () {
+                                datadelete(id,r.errcode,)
+                            });
+                        }
                     }
                 }
             })
         });
     }
+    //删除无效数据
+    function datadelete(taskid,code) {
+        if(code == ""){
+            alert("删除失败！")
+        }
+        $.ajax({
+            url: "/front/task/datadelete",
+            type: "post",
+            data: {"taskid": taskid,"code": code},
+            success: function (r) {
+                if (r&&r.ok){
+                    showTip("删除成功", 2000, function () { });
+                }else{
+                    showTip(r.msg, 3000, function () { })
+                }
+            }
+        })
+    }
 
     //数据推送
     function datasend(taskid) {
@@ -293,11 +324,16 @@
         var coll = $("#coll").val();
         var isbidding = $("#isbidding").val();
         var flows = [];
+        var checkfields = $("#checkfields").val();
         if (isbidding == "是"){
             if (coll != {{.T.datacoll}}){
                 alert("表名不是"+{{.T.datacoll}})
                 return;
             }
+            if (checkfields == ""){
+                alert("校验字段不能为空！")
+                return;
+            }
             $("#select4 option").each(function(i,val){
                 flows[i] = this.value
             })
@@ -318,7 +354,7 @@
         $.ajax({
             url: "/front/task/save",
             type: "post",
-            data: {"id":_id,"name": name, "coll": coll, "isbidding": isbidding, "flows": flows.join(",")},
+            data: {"id":_id,"name": name, "coll": coll,"checkfields":checkfields, "isbidding": isbidding, "flows": flows.join(",")},
             success: function (r) {
                 if (r.rep) {
                     $("#task_modal").modal("hide");
@@ -331,13 +367,14 @@
         })
     }
 
-    function edit(id,name,db,coll,isbidding,flows) {
+    function edit(id,name,db,coll,fields,isbidding,flows) {
         _id = id;
         $("#select3").empty();
         $("#select4").empty();
         $("#name").val(name);
         $("#db").val(db);
         $("#coll").val(coll);
+        $("#checkfields").val(fields);
         if (isbidding == "true"){
             $("#isbidding").val("是")
             $("#selectclear").css('display','inline-block')
@@ -404,11 +441,11 @@
                 showMsg("文件格式非法", function() {});
             }else{
                 $(this).parent().submit();
-                com.maskShow("正在导入数据");
+                showLoading("正在导入数据...")
                 var ret=setInterval(function(){
                     var f=$(window.frames[0].document).find("form");
+                    hideLoading();
                     if(f.length==0){
-                         com.maskHide();
                          var b=$(window.frames[0].document).find("body").html();
                          showMsg(b, function() {});
                          $(window.frames[0].document).find("body").append("<form id='uploadform' method='post' enctype='multipart/form-data' action='/front/data/importdata'><input type='file' name='xlsx' /></form>");
@@ -422,12 +459,34 @@
                     }
                     clearInterval(ret);
                     ttable.ajax.reload();
-                },500)
+                },5000)
 
             }
         })
     }
 
+    function preparebidding() {
+        showConfirm("确认根据bidding_copy表信息获取对应bidding数据？", function() {
+            showLoading("正在准备数据...");
+            // $.post(
+            //     "/front/data/preparebidding",
+            //     {"codes":codes.join(","),"ids":ids.join(",")},
+            //     function(r){
+            //
+            //     }
+            // )
+            $.ajax({
+                url: "/front/data/preparebidding",
+                type: "post",
+                success: function (r) {
+                    hideLoading();
+                    showMsg(r.msg, function () {
+                        
+                    });
+                }
+            })
+        })
+    }
     //右移
     $("#selectclear2 #right2").click(function(){
         $("#select3 option:selected").appendTo("#select4");