站点更新修改

src/front/front.go

@@ -596,7 +596,12 @@ func (f *Front) ImportLua() {
 						}
 						o["infoformat"] = infoformat
 						//存储表
-						o["coll"] = cells[16].Value
+						coll := cells[16].Value
+						if strings.Contains(code, "bidding") {
+							errorinfo[cells[1].Value] = "第" + strconv.Itoa(k) + "行爬虫" + cells[1].Value + ",存储表错误"
+							continue
+						}
+						o["coll"] = coll
 						//table := cells[6].Value
 						//o["table"] = table
 						//o["transfercode"] = qu.IntAll(Transfercode[table])

src/front/site.go

@@ -20,7 +20,7 @@ type Site struct {
 	*xweb.Action
 	siteList        xweb.Mapper `xweb:"/center/site/sitelist"`        //控制中心
 	saveSite        xweb.Mapper `xweb:"/center/site/savesite"`        //新增站点
-	importFile      xweb.Mapper `xweb:"/center/site/importfile"`      //导入站点
+	importSite      xweb.Mapper `xweb:"/center/site/importsite"`      //导入站点
 	deleteSite      xweb.Mapper `xweb:"/center/site/delete"`          //删除站点
 	getLuas         xweb.Mapper `xweb:"/center/site/getluas/(.*)"`    //
 	getLuasBySite   xweb.Mapper `xweb:"/center/site/getluasbysite"`   //
@@ -115,15 +115,15 @@ func (s *Site) SaveSite() {
 	sitetype := s.GetString("sitetype")
 	psite := s.GetString("psite")
 	ssite := s.GetString("ssite")
-	tmpdomain, status, event, platform, tmparea, tmpcity, tmpdistrict, remarktime := util.GetLuasInfoBySite(site)
+	tmpdomain, status, event, platform, remarktime := util.GetLuasInfoBySite(site, area, city, district)
 	if domain == "" {
 		domain = tmpdomain
 	}
-	if area == "" {
-		area = tmparea
-		city = tmpcity
-		district = tmpdistrict
-	}
+	//if area == "" {
+	//	area = tmparea
+	//	city = tmpcity
+	//	district = tmpdistrict
+	//}
 	update := map[string]interface{}{
 		"site":         site,
 		"another_name": othername,
@@ -157,7 +157,7 @@ func (s *Site) SaveSite() {
 	s.ServeJson(map[string]interface{}{"ok": b})
 }
 
-func (s *Site) ImportFile() {
+func (s *Site) ImportSite() {
 	defer qu.Catch()
 	msg := ""
 	file, _, err := s.GetFile("xlsx")
@@ -188,15 +188,15 @@ func (s *Site) ImportFile() {
 				site_type := r.Cells[7].Value
 				p_site := r.Cells[8].Value
 				s_site := r.Cells[9].Value
-				tmpdomain, status, event, platform, tmparea, tmpcity, tmpdistrict, remarktime := util.GetLuasInfoBySite(site)
+				tmpdomain, status, event, platform, remarktime := util.GetLuasInfoBySite(site, area, city, district)
 				if domain == "" {
 					domain = tmpdomain
 				}
-				if area == "" {
-					area = tmparea
-					city = tmpcity
-					district = tmpdistrict
-				}
+				//if area == "" {
+				//	area = tmparea
+				//	city = tmpcity
+				//	district = tmpdistrict
+				//}
 				save := map[string]interface{}{
 					"site":          site,
 					"another_name":  another_name,

src/taskManager/taskManager.go

@@ -134,7 +134,7 @@ func (t *TaskM) Taskfile() {
 							o["l_comeintime"] = time.Now().Unix()
 							o["i_event"] = (*lua)["event"]
 							o["i_times"] = 0
-							o["s_date"] = time.Now().Format("2006-01-02")
+							//o["s_date"] = time.Now().Format("2006-01-02")
 							o["s_source"] = "人工"
 							o["s_code"] = code
 							o["s_descript"] = cells[1].Value //描述
@@ -474,10 +474,13 @@ func (t *TaskM) SaveNewTask() {
 	newTask["s_code"] = code             //代码
 	newTask["i_state"] = 2               //完成状态
 	newTask["l_comeintime"] = comeintime //创建时间
-	newTask["s_date"] = time.Now().Format("2006-01-02")
+	//newTask["s_date"] = time.Now().Format("2006-01-02")
 	newTask["i_times"] = 0
 	newTask["i_pendstate"] = 0
 	newTask["l_checktime"] = time.Now().Unix()
+	newTask["i_frequencyerrtimes"] = time.Now().Unix() //采集频率异常次数
+	newTask["s_platform"] = "golua平台"
+	newTask["i_num"] = 0
 	queryL := map[string]interface{}{
 		"code": code,
 	}

src/timetask/timetask.go

@@ -73,7 +73,7 @@ func UpdateSiteInfo() {
 	sites, _ := util.MgoEB.Find(sp.Config.SiteColl, ``, ``, `{"site":1}`, false, -1, -1)
 	for _, s := range *sites {
 		site := qu.ObjToString(s["site"])
-		domain, status, event, platform, _, _, _, _ := util.GetLuasInfoBySite(site)
+		domain, status, event, platform, _ := util.GetLuasInfoBySite(site, "", "", "")
 		set := map[string]interface{}{
 			"$set": map[string]interface{}{
 				"platform":      platform,

src/util/util.go

@@ -299,18 +299,41 @@ func SpiderPassCheckListAndDetail_back(list []map[string]interface{}, data map[s
 	return
 }
 
-func GetLuasInfoBySite(site string) (domain, status, event, platform, area, city, district string, remarktime int64) {
+func GetLuasInfoBySite(site, area, city, district string) (domain, status, event, platform string, remarktime int64) {
 	shelveUp := 0
 	eventMap, platformMap := map[int]interface{}{}, map[string]interface{}{}
 	eventArr, platformArr := []string{}, []string{}
-	areaMap := map[string]int{}
-	areaCityMap := map[string]map[string]int{}
-	cityDistrictMap := map[string]map[string]int{}
+	//areaMap := map[string]int{}
+	//areaCityMap := map[string]map[string]int{}
+	//cityDistrictMap := map[string]map[string]int{}
 	domainMap := map[string]int{}
 	remarktime = time.Now().Unix()
 	//luas, _ := MgoE.Find("luaconfig", `{"param_common.1":"`+site+`"}`, ``, `{"model":1,"event":1,"state":1,"platform":1,"param_common":1,"comeintime":1}`, false, -1, -1)
-	luas, _ := MgoEB.Find("luaconfig", `{"param_common.1":"`+site+`"}`, ``, `{"model":1,"event":1,"state":1,"platform":1,"param_common":1,"comeintime":1}`, false, -1, -1)
+	luas, _ := MgoEB.Find("luaconfig", `{"site":"`+site+`"}`, ``, `{"model":1,"event":1,"state":1,"platform":1,"param_common":1,"comeintime":1}`, false, -1, -1)
+	arr := [][]map[string]interface{}{}
 	for _, l := range *luas {
+		update := []map[string]interface{}{}
+		set := map[string]interface{}{}
+		//更新爬虫area、city、district
+		if area != "" {
+			set["model.area"] = area
+		}
+		if area == "全国" {
+			set["model.city"] = ""
+			set["model.district"] = ""
+		} else if area != "" {
+			if city != "" {
+				set["model.city"] = city
+			}
+			if district != "" {
+				set["model.district"] = district
+			}
+		}
+		if len(set) > 0 {
+			update = append(update, map[string]interface{}{"_id": l["_id"]})
+			update = append(update, map[string]interface{}{"$set": set})
+			arr = append(arr, update)
+		}
 		//remarktime
 		if comeintime := qu.Int64All(l["comeintime"]); comeintime != int64(0) && comeintime < remarktime {
 			remarktime = comeintime
@@ -335,29 +358,29 @@ func GetLuasInfoBySite(site string) (domain, status, event, platform, area, city
 		eventMap[event] = true
 		platformMap[platform] = true
 		//area、city、district
-		if model, ok := l["model"].(map[string]interface{}); ok && model != nil {
-			a := qu.ObjToString(model["area"])
-			c := qu.ObjToString(model["city"])
-			d := qu.ObjToString(model["district"])
-			if a != "" {
-				areaMap[a] = areaMap[a] + 1
-				if c != "" {
-					if cityNum := areaCityMap[a]; cityNum != nil {
-						cityNum[c] = cityNum[c] + 1
-					} else {
-						areaCityMap[a] = map[string]int{c: 1}
-					}
-					if d != "" {
-						if distrctNum := cityDistrictMap[c]; distrctNum != nil {
-							distrctNum[d] = distrctNum[d] + 1
-						} else {
-							cityDistrictMap[c] = map[string]int{d: 1}
-						}
-					}
-				}
-
-			}
-		}
+		//if model, ok := l["model"].(map[string]interface{}); ok && model != nil {
+		//	a := qu.ObjToString(model["area"])
+		//	c := qu.ObjToString(model["city"])
+		//	d := qu.ObjToString(model["district"])
+		//	if a != "" {
+		//		areaMap[a] = areaMap[a] + 1
+		//		if c != "" {
+		//			if cityNum := areaCityMap[a]; cityNum != nil {
+		//				cityNum[c] = cityNum[c] + 1
+		//			} else {
+		//				areaCityMap[a] = map[string]int{c: 1}
+		//			}
+		//			if d != "" {
+		//				if distrctNum := cityDistrictMap[c]; distrctNum != nil {
+		//					distrctNum[d] = distrctNum[d] + 1
+		//				} else {
+		//					cityDistrictMap[c] = map[string]int{d: 1}
+		//				}
+		//			}
+		//		}
+		//
+		//	}
+		//}
 	}
 	//
 	for e, _ := range eventMap {
@@ -377,30 +400,35 @@ func GetLuasInfoBySite(site string) (domain, status, event, platform, area, city
 		}
 	}
 	status = fmt.Sprintf("%d%s%d", shelveUp, "/", len(*luas))
-	//
-	an, cn, dn := 0, 0, 0
-	for at, num := range areaMap {
-		if num > an {
-			area = at
-			an = num
-		}
-	}
-	if area != "" {
-		for ct, num := range areaCityMap[area] {
-			if num > cn {
-				city = ct
-				cn = num
-			}
-		}
-	}
-	if city != "" {
-		for dt, num := range cityDistrictMap[city] {
-			if num > dn {
-				district = dt
-				dn = num
-			}
-		}
+	//批量更新
+	if len(arr) > 0 {
+		MgoEB.UpdateBulk("luaconfig", arr...)
+		arr = [][]map[string]interface{}{}
 	}
+
+	//an, cn, dn := 0, 0, 0
+	//for at, num := range areaMap {
+	//	if num > an {
+	//		area = at
+	//		an = num
+	//	}
+	//}
+	//if area != "" {
+	//	for ct, num := range areaCityMap[area] {
+	//		if num > cn {
+	//			city = ct
+	//			cn = num
+	//		}
+	//	}
+	//}
+	//if city != "" {
+	//	for dt, num := range cityDistrictMap[city] {
+	//		if num > dn {
+	//			district = dt
+	//			dn = num
+	//		}
+	//	}
+	//}
 	return
 }
 

src/web/templates/head.html

@@ -598,7 +598,7 @@
 		{{if eq (session "platform") "golua平台"}}
      	<li data="index_heart"><a href="/center/heart"><i class="fa fa-heartbeat"></i> <span>心跳监控</span></a></li>
     	<li data="index_errlua"><a href="/center/errorLua"><i class="fa fa-warning"></i> <span>错误爬虫管理</span></a></li>
-    	<li data="index_errdata"><a href="/center/errorData"><i class="glyphicon glyphicon-exclamation-sign"></i> <span>错误信息管理</span></a></li>
+<!--    	<li data="index_errdata"><a href="/center/errorData"><i class="glyphicon glyphicon-exclamation-sign"></i> <span>错误信息管理</span></a></li>-->
 <!--		<li data="index_wtgl"><a href="/center/managerQues"><i class="glyphicon glyphicon-question-sign"></i> <span>问题管理</span></a></li>-->
 		{{end}}
 		<li data="index_site"><a href="/center/site/sitelist"><i class="glyphicon glyphicon-home"></i> <span>站点列表</span></a></li>

src/web/templates/sitelist.html

@@ -80,7 +80,7 @@
                             <div class="form-group">
                                 <div class="col-sm-offset-2 col-sm-10 operateStyle">
                                     <input type="button" id="saveSiteId" onclick="updateSite()" class="btn btn-primary" value="保存">
-                                    <input type="button" onclick="cancelAdd()" class="btn btn-default" value="取消">
+                                    <input type="button" data-dismiss="modal" class="btn btn-default" value="取消">
                                 </div>
                             </div>
                         </form>
@@ -192,8 +192,7 @@
          <a class="btn btn-default btn-sm" id="add" onclick="addSite()">新增</a>
          <a class="btn btn-default btn-sm" onclick="del()">删除</a>
 <!--			{{end}}-->
-         <iframe srcdoc="<form id='uploadform' method='post' enctype='multipart/form-data' action='/center/site/importf
-				ile'><input type='file' name='xlsx' /></form>" height=0 scrolling=no class="hide"  id="fileframe">
+         <iframe srcdoc="<form id='uploadform' method='post' enctype='multipart/form-data' action='/center/site/importsite'><input type='file' name='xlsx' /></form>" height=0 scrolling=no class="hide"  id="fileframe">
          </iframe>
 	 </small>
    </h1>
@@ -587,7 +586,7 @@ function importSite(){
                 if(f.length==0){
                     common.maskHide();
                     var b=$(window.frames[0].document).find("body").text();
-                    $(window.frames[0].document).find("body").append("<form id='uploadform' method='post' enctype='multipart/form-data' action='/center/site/importfile'><input type='file' name='xlsx' /></form>");
+                    $(window.frames[0].document).find("body").append("<form id='uploadform' method='post' enctype='multipart/form-data' action='/center/site/importsite'><input type='file' name='xlsx' /></form>");
                     if(b!=""&&b!=`""`){
                         showTip(b, 2000);
                     }else{