'xg'

src/main.go

@@ -15,6 +15,7 @@ import (
 	"runtime"
 	. "spiderutil"
 	"time"
+
 	//"qfw/util/redis"
 	"github.com/donnie4w/go-logger/logger"
 	"github.com/go-xweb/xweb"
@@ -121,7 +122,7 @@ func main() {
 	//批量保存错误数据
 	go spider.UpdateErrDataMgo()
 	//保存爬虫采集非本站点数据
-	go spider.SaveOtherSiteData()
+	//go spider.SaveOtherSiteData()
 	//批量保存心跳信息
 	go spider.UpdateHeartInfo()
 	logger.Debug(Config.Webport)

src/spider/spider.go

@@ -20,14 +20,15 @@ import (
 	//mgu "qfw/util/mongodbutil"
 	//"qfw/util/redis"
 
-	"github.com/donnie4w/go-logger/logger"
-	"github.com/yuin/gopher-lua"
 	es "qfw/util/elastic"
 	"regexp"
 	util "spiderutil"
 	"strings"
 	"sync/atomic"
 	"time"
+
+	"github.com/donnie4w/go-logger/logger"
+	"github.com/yuin/gopher-lua"
 )
 
 type Heart struct {
@@ -290,7 +291,7 @@ func (s *Spider) DownListPageItem() (errs interface{}) {
 				for i := 1; i <= tabLen; i++ {
 					v := tbl.RawGetInt(i).(*lua.LTable)
 					tmp := util.TableToMap(v)
-					s.ThisSiteData(tmp)      //统计当前下载数据是否是本站点数据
+					//s.ThisSiteData(tmp)      //统计当前下载数据是否是本站点数据
 					if !s.IsHistoricalMend { //不是历史补漏
 						tmp["dataging"] = 0 //数据中打标记dataging=0
 						if s.DownDetail {