功能调整

src/saveServer/httpserver.go

@@ -4,6 +4,7 @@ import (
 	"html/template"
 	"log"
 	"net/http"
+	qutil "qfw/util"
 )
 
 const (
@@ -38,41 +39,48 @@ func HttpServer(port string) {
 			StopFlag = 1
 			//yq
 			saveyqlock.Lock()
+			qutil.Debug("SaveYqCache:", len(SaveYqCache))
 			if len(SaveYqCache) > 0 {
 				saveYqMust()
 			}
 			saveyqlock.Unlock()
 			//other
 			saveotherlock.Lock()
+			qutil.Debug("SaveOtherCache:", len(SaveOtherCache))
 			if len(SaveOtherCache) > 0 {
 				saveOtherMust()
 			}
 			saveotherlock.Unlock()
 			//bidding
 			savelock.Lock()
+			qutil.Debug("SaveCache:", len(SaveCache))
 			if len(SaveCache) > 0 {
 				saveMust()
 			}
 			savelock.Unlock()
 			//bidding_file
 			savefilelock.Lock()
+			qutil.Debug("SaveFileCache:", len(SaveFileCache))
 			if len(SaveFileCache) > 0 {
 				saveFileMust()
 			}
 			savefilelock.Unlock()
 			//bidding_news
+			qutil.Debug("SaveNewsCache:", len(SaveNewsCache))
 			saveNewsLock.Lock()
 			if len(SaveNewsCache) > 0 {
 				saveNewsMust()
 			}
 			saveNewsLock.Unlock()
 			//update
+			qutil.Debug("UpdateCache:", len(UpdateCache))
 			updatelock.Lock()
 			if len(UpdateCache) > 0 {
 				updateMust()
 			}
 			updatelock.Unlock()
 			//
+			qutil.Debug("saveCommMust:", mnum)
 			mlock.Lock()
 			if mnum > 0 {
 				saveCommMust()

src/saveServer/main.go

@@ -96,7 +96,7 @@ func inits() {
 	go SaveRepeat() //记录重复数据
 	go TimerSave()  //定时保存数据
 	//go TimingTask()    //定时发邮件
-	//go TimingSaveNum() //定时检测半小时内保存数据量
+	go TimingSaveNum() //定时检测半小时内保存数据量
 }
 
 var IS = &InfoStatus{time.Now().Unix(), 0, map[int]int{}, &sync.Mutex{}}

src/saveServer/processdata.go

@@ -63,34 +63,39 @@ func NewSaveBidding(tmp map[string]interface{}) (b bool, res int, mgoid, mgocoll
 	//publishtime校验、修正
 	nowTime := time.Now().Unix()
 	if tmp["publishtime"] != nil {
-		if publishtime_str, ok := tmp["publishtime"].(string); ok { //字段类型异常
-			errorData(LEVEL_ERROR, true, "publishtime", "Field Type Error", href, title, &warn, tmp)
-			//修正
-			pTmp, err := time.ParseInLocation(qutil.Date_Full_Layout, publishtime_str, time.Local)
-			if err != nil {
-				pTmp, err = time.ParseInLocation(qutil.Date_Short_Layout, publishtime_str, time.Local)
-				if err != nil {
-					tmp["publishtime"] = nowTime
+		publishtime := nowTime
+		if publishtime_str, ok := tmp["publishtime"].(string); ok { //字段类型异常的发布时间
+			if publishtime_str == "-1" {
+				publishtime = -1
+			} else {
+				pTmp, err := time.ParseInLocation(qutil.Date_Full_Layout, publishtime_str, time.Local)
+				if err == nil {
+					publishtime = pTmp.Unix()
 				} else {
-					tmp["publishtime"] = pTmp.Unix()
+					pTmp, err = time.ParseInLocation(qutil.Date_Short_Layout, publishtime_str, time.Local)
+					if err == nil {
+						publishtime = pTmp.Unix()
+					} else {
+						errorData(LEVEL_ERROR, true, "publishtime", "Field Type Error", href, title, &warn, tmp)
+					}
 				}
-			} else {
-				tmp["publishtime"] = pTmp.Unix()
-			}
-		} else {
-			publishtime := qutil.Int64All(tmp["publishtime"])
-			if publishtime != -1 { //publishtime=-1的放行，结构化数据处理提取
-				if publishtime < 0 || (publishtime > 0 && publishtime < 1000000000) || publishtime > nowTime {
-					errorData(LEVEL_ERROR, true, "publishtime", "Publishtime Is Error", href, title, &warn, tmp)
-					tmp["publishtime"] = nowTime
-				} else if publishtime == 0 {
-					errorData(LEVEL_WARN, true, "publishtime", "Publishtime Is Zero", href, title, &warn, tmp)
-					tmp["publishtime"] = nowTime
-				} /*else if publishtime > nowTime {
-					errorData(LEVEL_ERROR, true, "publishtime", "Publishtime Is Error", href, title, &warn, tmp)
-					tmp["publishtime"] = nowTime
-				}*/
 			}
+		} else { //时间戳类型的发布时间
+			publishtime = qutil.Int64All(tmp["publishtime"])
+		}
+		tmp["publishtime"] = publishtime
+		//检测
+		if publishtime != -1 { //publishtime=-1的放行，结构化数据处理提取
+			if publishtime < 0 || (publishtime > 0 && publishtime < 1000000000) || publishtime > nowTime {
+				errorData(LEVEL_ERROR, true, "publishtime", "Publishtime Is Error", href, title, &warn, tmp)
+				tmp["publishtime"] = nowTime
+			} else if publishtime == 0 {
+				errorData(LEVEL_WARN, true, "publishtime", "Publishtime Is Zero", href, title, &warn, tmp)
+				tmp["publishtime"] = nowTime
+			} /*else if publishtime > nowTime {
+				errorData(LEVEL_ERROR, true, "publishtime", "Publishtime Is Error", href, title, &warn, tmp)
+				tmp["publishtime"] = nowTime
+			}*/
 		}
 	} else {
 		errorData(LEVEL_ERROR, true, "publishtime", "Field Value Is Null", href, title, &warn, tmp)

src/saveServer/savedata.go

@@ -64,6 +64,7 @@ var (
 // 批量保存舆情数据
 func saveYqMust() {
 	saveyqnum += len(SaveYqCache)
+	savenum += len(SaveYqCache)
 	tools.Mgo.SaveBulk(SaveYqColl, SaveYqCache...)
 	go log.Println("saveYqMust:", saveyqnum)
 	time.Sleep(time.Second * 2)
@@ -74,6 +75,7 @@ func saveYqMust() {
 // 批量保存临时数据
 func saveOtherMust() {
 	saveothernum += len(SaveOtherCache)
+	savenum += len(SaveOtherCache)
 	tools.Mgo.SaveBulk(SaveOtherColl, SaveOtherCache...)
 	go log.Println("saveOtherMust:", saveothernum)
 	time.Sleep(time.Second * 2)
@@ -105,6 +107,7 @@ func saveFileMust() {
 // 批量新闻信息
 func saveNewsMust() {
 	saveNewsNum += len(SaveNewsCache)
+	savenum += len(SaveNewsCache)
 	MgoNews.SaveBulk(SaveNewsColl, SaveNewsCache...)
 	go log.Println("saveNewsMust:", saveNewsNum)
 	time.Sleep(time.Second * 2)

src/saveServer/sendmail.go

@@ -31,14 +31,14 @@ func (m *myWrite) Write(p []byte) (n int, err error) {
 	return
 }
 
-//定时统计数据发邮件
+// 定时统计数据发邮件
 func TimingTask() {
 	cron.Start()
 	cron.AddFunc(fir_sendmailtime, func() { StatisticalData(70200) }) //第一次发邮件
 	cron.AddFunc(sec_sendmailtime, func() { StatisticalData(16200) }) //第二次发邮件
 }
 
-//定时检测半小时内保存数据量
+// 定时检测半小时内保存数据量
 func TimingSaveNum() {
 	log.Println("savenum:", savenum, "tmpsavenum:", tmpsavenum)
 	if savenum != 0 {
@@ -47,10 +47,10 @@ func TimingSaveNum() {
 		}
 	}
 	tmpsavenum = savenum
-	time.AfterFunc(time.Minute*30, TimingSaveNum)
+	time.AfterFunc(time.Minute*60, TimingSaveNum)
 }
 
-//统计数据
+// 统计数据
 func StatisticalData(t int64) {
 	defer qutil.Catch()
 	//qutil.Debug("------开始统计----")
@@ -96,7 +96,7 @@ func StatisticalData(t int64) {
 	}
 }
 
-//发邮件通知
+// 发邮件通知
 func SendMail(tmp map[string]map[string]int) {
 	//qutil.Debug("------开始发送邮件----")
 	file := xlsx.NewFile()