添加同步 高质量库 过滤字段

highMark/config.json

@@ -15,5 +15,6 @@
     "username": "",
     "password": ""
   },
-  "spec": "0 */1 * * * *"
+  "spec": "0 00 01 * * *",
+  "no_fields": "matchkey,matchkey,field_source,purchasinglist_alltag,info,jytest_href,matchkey"
 }

highMark/main.go

@@ -1,16 +1,19 @@
-package highMark
+package main
 
 import (
 	"fmt"
+	"github.com/cron"
 	"mongodb"
 	"os"
 	"qfw/util"
+	"strings"
 	"time"
 )
 
 var (
 	Mgo, MgoH                             *mongodb.MongodbSim
 	Sysconfig, bidddingConf, biddingHConf map[string]interface{}
+	noFields                              string
 )
 
 func Init() {
@@ -40,24 +43,23 @@ func Init() {
 	}
 
 	MgoH.InitPool()
-
+	noFields = util.ObjToString(Sysconfig["no_fields"])
 }
 
-//func main() {
-//	Init()
-//	c := cron.New()
-//	err := c.AddFunc(Sysconfig["spec"].(string), Mark)
-//	if err != nil {
-//		util.Debug("err", err)
-//	}
-//
-//	c.Start()
-//	defer c.Stop()
-//
-//	select {}
-//
-//	//highMark()
-//}
+func main() {
+	Init()
+	c := cron.New()
+	err := c.AddFunc(Sysconfig["spec"].(string), Mark)
+	if err != nil {
+		util.Debug("err", err)
+	}
+
+	c.Start()
+	defer c.Stop()
+
+	select {}
+
+}
 
 func Mark() {
 	go highMark()
@@ -115,10 +117,17 @@ func highMark() {
 			res := calculateFlag(taginfo, *fields) //返回标注的十进制数字
 
 			if data, ok := markedData["v_baseinfo"].(map[string]interface{}); ok {
-				data["_id"] = tmp["_id"]
+				data["_id"] = mongodb.StringTOBsonId(util.ObjToString(data["id"]))
 				data["field_bitvalue"] = res
 				data["i_comeintime"] = time.Now().Unix()
 				data["i_updatetime"] = time.Now().Unix()
+				//删除多余无用字段
+				noField := strings.Split(noFields, ",")
+				if len(noField) > 0 {
+					for _, field := range noField {
+						delete(data, field)
+					}
+				}
 
 				update := make(map[string]interface{})
 				update["$set"] = data
@@ -174,7 +183,7 @@ func highMark() {
 	util.Debug("所有任务处理完毕")
 }
 
-//calculateFlag 根据数据，返回被标注的字段数字
+// calculateFlag 根据数据，返回被标注的字段数字
 func calculateFlag(marked map[string]interface{}, data []map[string]interface{}) uint64 {
 	var result uint64
 	for _, item := range data {