editor库迁移163

src/config.json

@@ -2,8 +2,6 @@
     "webport": "7100",
     "mongodb_spider": "192.168.3.207:27092",
     "spider_dbsize": 50,
-    "mongodb_editor": "192.168.3.207:27092",
-    "editor_dbsize": 2,
     "bideditor": {
         "addr": "192.168.3.207:27001",
         "db": "editor",

src/main.go

@@ -90,12 +90,12 @@ func InitMgo() {
 		DbName:      "spider",
 	}
 	spider.MgoS.InitPool()
-	spider.MgoE = &mgo.MongodbSim{
-		MongodbAddr: Config.Mongodb_editor,
-		Size:        Config.Editor_dbsize,
-		DbName:      "editor",
-	}
-	spider.MgoE.InitPool()
+	//spider.MgoE = &mgo.MongodbSim{
+	//	MongodbAddr: Config.Mongodb_editor,
+	//	Size:        Config.Editor_dbsize,
+	//	DbName:      "editor",
+	//}
+	//spider.MgoE.InitPool()
 	spider.MgoEB = &mgo.MongodbSim{
 		MongodbAddr: Config.BidEditor.Addr,
 		Size:        Config.BidEditor.Size,

src/spider/handler.go

@@ -1342,7 +1342,7 @@ func SpiderCodeSendToEditor(code string) {
 		}
 	}
 	logger.Info("Code:", code, "	Send Move Event:", ok)
-	MgoE.Save("luamovelog", map[string]interface{}{
+	MgoEB.Save("luamovelog", map[string]interface{}{
 		"code":       code,
 		"comeintime": time.Now().Unix(),
 		"type":       "sendfail",

src/spider/spider.go

@@ -90,7 +90,8 @@ var Es *es.Elastic
 var EsIndex string
 var EsType string
 var MgoS *mgo.MongodbSim
-var MgoE *mgo.MongodbSim
+
+//var MgoE *mgo.MongodbSim
 var MgoEB *mgo.MongodbSim
 var UpdataMgoCache = make(chan []map[string]interface{}, 1000)   //更新要重下数据的状态
 var UpdataHeartCache = make(chan []map[string]interface{}, 1000) //更新爬虫心跳信息
@@ -406,7 +407,7 @@ func (s *Spider) DownListPageItem() (errs interface{}) {
 		downtimes = 0 //当前页下载无误，重置下载重试次数
 		util.TimeSleepFunc(100*time.Millisecond, TimeSleepChan)
 	}
-	logger.Info(s.Code, "本轮列表页采集详情：", downloadAllNum, repeatAllNum, start)
+	logger.Info(s.Code, "本轮列表页采集详情：", downloadAllNum, repeatAllNum, start, s.Stop)
 	if !util.Config.IsHistoryEvent && !s.Stop { //非历史节点统计下载率
 		nowTime := time.Now()
 		sDate := qu.FormatDate(&nowTime, qu.Date_Short_Layout)

src/spider/store.go

@@ -198,6 +198,7 @@ func GcCount() {
 func SaveErrorData(modifyuser string, pd map[string]interface{}, err interface{}) {
 	defer util.Catch()
 	if href := util.ObjToString(pd["href"]); href != "" {
+		delete(pd, "_id")
 		pd["state"] = 0
 		pd["from"] = "lua"
 		pd["comeintime"] = time.Now().Unix()