定时-备份

udpfilterdup/src/datamap.go

@@ -67,7 +67,7 @@ type historymap struct {
 
 func TimedTaskDatamap(days int,lasttime int64) *datamap {
 	log.Println("数据池开始重新构建")
-	datelimit = qutil.Float64All(1 * 86400)
+	datelimit = qutil.Float64All(days * 86400)
 	dm := &datamap{sync.Mutex{}, days, map[string][]*Info{}, []string{}, []string{},map[string]bool{}}
 	if lasttime <0 {
 		log.Println("数据池空数据")

udpfilterdup/src/main.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"github.com/cron"
 	"log"
 	mu "mfw/util"
 	"net"
@@ -17,7 +18,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/cron"
 	"gopkg.in/mgo.v2/bson"
 )
 
@@ -587,8 +587,10 @@ func historyTask(data []byte, mapInfo map[string]interface{}) {
 							})
 						}
 					} else { //高质量数据
+
 						basic_bool := basicDataScore(source, info)
 						if !basic_bool {
+
 							HM.replaceSourceData(info, source.id) //替换
 							repeat_idMap["_id"] = StringTOBsonId(source.id)
 							repeat_id = info.id
@@ -792,8 +794,8 @@ func timedTaskOnce() {
 			}
 
 			b, source, reason := DM.check(info)
-			//log.Println("判重结果", b, reason)
 			if b { //有重复,生成更新语句，更新抽取和更新招标
+				log.Println("判重结果", b, reason)
 				repeateN++
 				var is_replace = false
 				var mergeArr = []int64{}                    //更改合并数组记录
@@ -872,6 +874,7 @@ func timedTaskOnce() {
 				} else { //高质量数据
 					basic_bool := basicDataScore(source, info)
 					if !basic_bool {
+						log.Println("高质量数据替换：",source.id,info.id)
 						DM.replaceSourceData(info, source.id) //替换
 						repeat_idMap["_id"] = StringTOBsonId(source.id)
 						repeat_id = info.id