特殊爬虫 适配

src/README.md

@@ -1,57 +1,55 @@
-
 {
-    "udpport": ":19097",
-    "dupdays": 7,
-    "mongodb": {
-        "addr": "172.17.4.85:27080",
-        "pool": 10,
-        "db": "qfw",
-        "extract": "result_20200715",
-        "extract_back": "result_20200714",
-        "site": {
-            "dbname": "qfw",
-            "coll": "site"
-        }
-    },
-    "task_mongodb": {
-        "task_addrName": "172.17.4.187:27081",
-        "task_dbName": "qfw",
-        "task_collName": "ocr_flie_over",
-        "pool": 5
-        
-    },
-    "jkmail": {
-        "to": "zhengkun@topnet.net.cn,zhangjinkun@topnet.net.cn",
-        "api": "http://10.171.112.160:19281/_send/_mail"
-    },
-    "nextNode": [
-        {
-            "addr": "172.17.4.194",
-            "port": 1782,
-            "stype": "project",
-            "memo": "合并项目"
-        },
-        {
-            "addr": "127.0.0.1",
-            "port": 1783,
-            "stype": "bidding",
-            "memo": "创建招标数据索引new"
-        }
-    ],
-    "threads": 1,
-    "lowHeavy":true,
-    "timingTask":false,
-    "timingSpanDay": 5,
-    "timingPubScope": 720,
-    "specialwords": "(重招|重新招标|勘察|施工|监理|总承包|土石方|可研)",
-    "specialtitle_0": "(包|标段|标包)[(（]?[0-9a-zA-Z一二三四五六七八九十零１２３４５６７８９][)）]?",
-    "specialtitle_1": "[0-9a-zA-Z一二三四五六七八九十零１２３４５６７８９](次|包|标段|标包|批|期)",
-    "specialtitle_2": "项目[(（][0-9a-zA-Z一二三四五六七八九十零１２３４５６７８９][)）]",
-    "beifen": "[(（]?[0-9一二三四五六七八九十零１２３４５６７８９再][)）]?[子分]?[次批标包]|重招|重新招标|勘察|设计|施工|监理|总承包|土石方|可研"
+"udpport": ":1785",
+"dupdays": 7,
+"mongodb": {
+"addr": "172.17.4.85:27080",
+"pool": 10,
+"db": "qfw",
+"extract": "result_20220219",
+"extract_back": "result_20220218",
+"extract_log": "result_replace_log"
+},
+"task_mongodb": {
+"task_addr": "172.17.4.187:27082,172.17.145.163:27083",
+"task_db": "qfw",
+"task_coll": "bidding_processing_ids",
+"task_bidding": "bidding",
+"task_pool": 5
+},
+"spider_mongodb": {
+"spider_addr": "172.17.4.87:27080",
+"spider_db": "editor",
+"spider_coll": "site",
+"spider_pool": 5
+},
+"userName": "zhengkun",
+"passWord": "zk@123123",
+"jkmail": {
+"to": "zhengkun@topnet.net.cn,wangjianghan@topnet.net.cn",
+"api": "http://172.17.145.179:19281/_send/_mail"
+},
+"nextNode": [
+{
+"addr": "172.17.4.196",
+"port": 1787,
+"stype": "bidding",
+"memo": "同步程序id段udp"
+}
+],
+"jyfb_data": [
+"a_jyxxfbpt_gg"
+],
+"threads": 4,
+"lowHeavy":true,
+"timingTask":false,
+"timingSpanDay": 5,
+"timingPubScope": 1440,
+"specialwords": "(重招|重新招标|勘察|施工|监理|总承包|土石方|可研)",
+"specialtitle_0": "(包|标段|标包)[(（]?[0-9a-zA-Z一二三四五六七八九十零１２３４５６７８９][)）]?",
+"specialtitle_1": "[0-9a-zA-Z一二三四五六七八九十零１２３４５６７８９](次|包|标段|标包|批|期)",
+"specialtitle_2": "项目[(（][0-9a-zA-Z一二三四五六七八九十零１２３４５６７８９][)）]",
+"beifen": "[(（]?[0-9一二三四五六七八九十零１２３４５６７８９再][)）]?[子分]?[次批标包]|重招|重新招标|勘察|设计|施工|监理|总承包|土石方|可研"
 }
-
-
-
 
 
 

src/config.json

@@ -1,37 +1,48 @@
 {
-    "udpport": ":17859",
+    "udpport": ":1785",
     "dupdays": 7,
     "mongodb": {
-        "addr": "127.0.0.1:27017",
-        "pool": 10,
-        "db": "zhengkun",
-        "extract": "repeat_test",
-        "extract_back": "repeat_test",
+        "addr": "192.168.3.206:27002",
+        "db": "qfw_data",
+        "username": "root",
+        "password": "root",
+        "extract": "result_20230510",
+        "extract_back": "result_20230510",
         "extract_log": "result_replace_log",
-        "site": {
-            "dbname": "zhengkun",
-            "coll": "site"
-        }
+        "pool": 5
     },
     "task_mongodb": {
-        "task_addrName": "127.0.0.1:27017",
-        "task_dbName": "zhengkun",
-        "task_collName": "repeat_test",
+        "task_addr": "192.168.3.206:27002",
+        "task_db": "qfw_data",
+        "username": "root",
+        "password": "root",
+        "task_coll": "bidding_processing_ids",
         "task_bidding": "bidding",
-        "pool": 10
+        "task_pool": 5
+    },
+    "spider_mongodb": {
+        "spider_addr": "192.168.3.166:27082",
+        "spider_db": "zk_extract_service",
+        "username": "",
+        "password": "",
+        "spider_coll": "site",
+        "spider_pool": 5
     },
     "jkmail": {
         "to": "zhengkun@topnet.net.cn,zhangjinkun@topnet.net.cn",
         "api": "http://172.17.145.179:19281/_send/_mail"
     },
     "nextNode": [
-
+        {
+            "addr": "127.0.0.1",
+            "port": 1783,
+            "stype": "bidding",
+            "memo": "创建招标数据索引new"
+        }
     ],
     "jyfb_data": [
         "a_jyxxfbpt_gg"
     ],
-    "userName": "",
-    "password": "",
     "threads": 1,
     "lowHeavy":true,
     "timingTask":false,

src/dataMethod.go

@@ -272,7 +272,16 @@ func leadingElementSame(v *Info, info *Info) bool {
 		isok++
 	}
 
-	if isok >= 5 {
+	if isok >= 5 { //加一层金额单位的逻辑校验
+		if v.budget != info.budget && v.budget != 0 && info.budget != 0 {
+			return false
+		}
+		if isBidWinningAmount(v.bidamount, info.bidamount) && v.bidamount != 0.0 && info.bidamount != 0.0 {
+			return false
+		}
+		if v.winner != "" && info.winner != "" && v.winner != info.winner {
+			return false
+		}
 		return true
 	}
 

src/datamap.go

@@ -74,8 +74,9 @@ func TimedTaskDatamap(days int, lasttime int64, numIndex int) *datamap {
 		//if n%10000 == 0 {
 		//	log.Println("当前 n:", n, "数量：", continuSum, tmp["_id"], tmp["publishtime"])
 		//}
-		if qutil.IntAll(tmp["repeat"]) == 1 || qutil.IntAll(tmp["repeat"]) == -1 ||
-			qutil.IntAll(tmp["dataging"]) == 1 || qutil.ObjToString(tmp["subtype"]) == "拟建" {
+		if qutil.IntAll(tmp["repeat"]) == 1 || qutil.IntAll(tmp["repeat"]) == -1 || qutil.IntAll(tmp["dataging"]) == 1 ||
+			qutil.ObjToString(tmp["subtype"]) == "拟建" || qutil.ObjToString(tmp["subtype"]) == "产权" ||
+			qutil.ObjToString(tmp["spidercode"]) == "sdxzbiddingsjzypc" {
 
 		} else {
 			if fmt.Sprint(reflect.TypeOf(tmp["publishtime"])) == "string" {
@@ -143,8 +144,9 @@ func NewDatamap(days int, lastid string) *datamap {
 	nowTime := time.Now().Unix() //当前时间的时间戳
 	n, continuSum := 0, 0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); n++ {
-		if qutil.IntAll(tmp["repeat"]) == 1 || qutil.IntAll(tmp["repeat"]) == -1 || qutil.ObjToString(tmp["subtype"]) == "拟建" {
-
+		if qutil.IntAll(tmp["repeat"]) == 1 || qutil.IntAll(tmp["repeat"]) == -1 ||
+			qutil.ObjToString(tmp["subtype"]) == "拟建" || qutil.ObjToString(tmp["subtype"]) == "产权" ||
+			qutil.ObjToString(tmp["spidercode"]) == "sdxzbiddingsjzypc" {
 		} else {
 			if fmt.Sprint(reflect.TypeOf(tmp["publishtime"])) == "string" {
 				continue
@@ -198,7 +200,6 @@ func NewInfo(tmp map[string]interface{}) *Info {
 		subtype == "竞谈" || subtype == "竞价" {
 		subtype = "招标"
 	}
-
 	area := qutil.ObjToString(tmp["area"])
 	if area == "A" {
 		area = "全国"

src/fullDataRepeat.go

@@ -59,10 +59,13 @@ func dealWithfullData(query map[string]interface{}, time_str string) {
 	total, isok, repeatN := 0, 0, 0
 	dataAllDict := make(map[string][]map[string]interface{}, 0)
 	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
-		if qu.IntAll(tmp["repeat"]) == 1 || qu.IntAll(tmp["repeat"]) == -1 || qu.ObjToString(tmp["subtype"]) == "拟建" {
+		if qu.IntAll(tmp["repeat"]) == 1 || qu.IntAll(tmp["repeat"]) == -1 ||
+			qu.ObjToString(tmp["subtype"]) == "拟建" || qu.ObjToString(tmp["subtype"]) == "产权" ||
+			qu.ObjToString(tmp["spidercode"]) == "sdxzbiddingsjzypc" {
 			tmp = make(map[string]interface{})
 			continue
 		}
+
 		isok++
 		subtype := qu.ObjToString(tmp["subtype"])
 		if subtype == "招标" || subtype == "邀标" || subtype == "询价" ||

src/historyRepeat.go

@@ -34,7 +34,7 @@ func historyRepeat() {
 		task_sess := task_mgo.GetMgoConn()
 		defer task_mgo.DestoryMongoConn(task_sess)
 		q := map[string]interface{}{}
-		it_last := task_sess.DB(task_mgo.DbName).C(task_collName).Find(&q).Sort("-_id").Iter()
+		it_last := task_sess.DB(task_mgo.DbName).C(task_coll).Find(&q).Sort("-_id").Iter()
 		isRepeatStatus := false
 		for tmp := make(map[string]interface{}); it_last.Next(&tmp); {
 			is_repeat_status := util.IntAll(tmp["repeat_status"])
@@ -50,16 +50,16 @@ func historyRepeat() {
 		}
 		if !isRepeatStatus {
 			log.Println("查询不到有标记的lteid数据")
-			log.Println("睡眠5分钟 gtid：", gtid, "lteid：", lteid)
-			time.Sleep(5 * time.Minute)
+			log.Println("睡眠2分钟 gtid：", gtid, "lteid：", lteid)
+			time.Sleep(2 * time.Minute)
 			continue
 		}
-		log.Println("查询完毕-找到有标记的lteid-先睡眠5分钟", gtid, lteid)
 
+		log.Println("查询完毕-找到有标记的lteid-先睡眠2分钟", gtid, lteid)
 		if isUpdateSite {
 			initSite()
 		}
-		time.Sleep(5 * time.Minute)
+		time.Sleep(2 * time.Minute)
 
 		sess := data_mgo.GetMgoConn() //连接器
 		defer data_mgo.DestoryMongoConn(sess)
@@ -83,7 +83,8 @@ func historyRepeat() {
 			//取-符合-发布时间X年内的数据
 			if util.IntAll(tmp["dataging"]) == 1 {
 				pubtime := util.Int64All(tmp["publishtime"])
-				if pubtime > 0 && pubtime >= between_time && qu.ObjToString(tmp["subtype"]) != "拟建" {
+				if pubtime > 0 && pubtime >= between_time && qu.ObjToString(tmp["subtype"]) != "拟建" && qu.ObjToString(tmp["subtype"]) != "产权" &&
+					qu.ObjToString(tmp["spidercode"]) != "sdxzbiddingsjzypc" {
 					oknum++
 					if deterTime == 0 {
 						log.Println("找到第一条符合条件的数据")

src/increaseRepeat.go

@@ -41,7 +41,11 @@ func increaseRepeat(mapInfo map[string]interface{}) {
 			tmp = make(map[string]interface{})
 			continue
 		}
-		if qu.ObjToString(tmp["subtype"]) == "拟建" {
+		if qu.ObjToString(tmp["subtype"]) == "拟建" || qu.ObjToString(tmp["subtype"]) == "产权" {
+			tmp = make(map[string]interface{})
+			continue
+		}
+		if qu.ObjToString(tmp["spidercode"]) == "sdxzbiddingsjzypc" {
 			tmp = make(map[string]interface{})
 			continue
 		}
@@ -202,7 +206,7 @@ func updateProcessUdpIdsInfo(sid string, eid string) {
 			"$lte": eid,
 		},
 	}
-	datas, _ := task_mgo.Find(task_collName, query, nil, nil)
+	datas, _ := task_mgo.Find(task_coll, query, nil, nil)
 	if len(datas) > 0 {
 		log.Println("开始更新流程段落记录~~", len(datas), "段")
 		for _, v := range datas {
@@ -215,7 +219,7 @@ func updateProcessUdpIdsInfo(sid string, eid string) {
 						"updatetime":    time.Now().Unix(),
 					},
 				}
-				task_mgo.UpdateById(task_collName, up_id, update)
+				task_mgo.UpdateById(task_coll, up_id, update)
 				log.Println("流程段落记录~~更新完毕~", update)
 			}
 		}
@@ -231,7 +235,7 @@ func updateOcrFileData(cur_lteid string) {
 	task_sess := task_mgo.GetMgoConn()
 	defer task_mgo.DestoryMongoConn(task_sess)
 	q_task := map[string]interface{}{}
-	it_last := task_sess.DB(task_mgo.DbName).C(task_collName).Find(&q_task).Sort("-_id").Iter()
+	it_last := task_sess.DB(task_mgo.DbName).C(task_coll).Find(&q_task).Sort("-_id").Iter()
 	isUpdateOcr := false
 	updateOcrFile := [][]map[string]interface{}{}
 	for tmp := make(map[string]interface{}); it_last.Next(&tmp); {
@@ -261,7 +265,7 @@ func updateOcrFileData(cur_lteid string) {
 		log.Println("出现异常问题，查询不到ocr的lteid", cur_lteid)
 	} else {
 		if len(updateOcrFile) > 0 {
-			task_mgo.UpSertBulk(task_collName, updateOcrFile...)
+			task_mgo.UpSertBulk(task_coll, updateOcrFile...)
 		}
 	}
 }

src/initData.go

@@ -0,0 +1,111 @@
+package main
+
+import (
+	"github.com/cron"
+	"github.com/go-xweb/log"
+	"nsqdata"
+	qu "qfw/util"
+	"regexp"
+)
+
+func initMgo() {
+	jyfb_arr := qu.ObjArrToStringArr(Sysconfig["jyfb_data"].([]interface{}))
+	jyfb_data = make(map[string]string, 0)
+	for _, v := range jyfb_arr {
+		jyfb_data[v] = v
+	}
+	log.Println("伪判重~", jyfb_data)
+	spider_mconf := Sysconfig["spider_mongodb"].(map[string]interface{})
+	spider_mgo = &MongodbSim{
+		MongodbAddr: spider_mconf["spider_addr"].(string),
+		DbName:      spider_mconf["spider_db"].(string),
+		Size:        qu.IntAllDef(spider_mconf["spider_pool"], 5),
+		UserName:    spider_mconf["username"].(string),
+		Password:    spider_mconf["password"].(string),
+	}
+	spider_mgo.InitPool()
+	spider_coll = spider_mconf["spider_coll"].(string)
+
+	task_mconf := Sysconfig["task_mongodb"].(map[string]interface{})
+	task_mgo = &MongodbSim{
+		MongodbAddr: task_mconf["task_addr"].(string),
+		DbName:      task_mconf["task_db"].(string),
+		Size:        qu.IntAllDef(task_mconf["task_pool"], 10),
+		UserName:    task_mconf["username"].(string),
+		Password:    task_mconf["password"].(string),
+	}
+	task_mgo.InitPool()
+	task_coll = task_mconf["task_coll"].(string)
+	task_bidding = task_mconf["task_bidding"].(string)
+
+	nextNode = qu.ObjArrToMapArr(Sysconfig["nextNode"].([]interface{}))
+	mconf := Sysconfig["mongodb"].(map[string]interface{})
+	data_mgo = &MongodbSim{
+		MongodbAddr: mconf["addr"].(string),
+		DbName:      mconf["db"].(string),
+		Size:        qu.IntAllDef(mconf["pool"], 10),
+		UserName:    mconf["username"].(string),
+		Password:    mconf["password"].(string),
+	}
+	data_mgo.InitPool()
+
+	extract = mconf["extract"].(string)
+	extract_back = mconf["extract_back"].(string)
+	extract_log = mconf["extract_log"].(string)
+
+	FilterRegTitle = regexp.MustCompile(qu.ObjToString(Sysconfig["specialwords"]))
+	FilterRegTitle_0 = regexp.MustCompile(qu.ObjToString(Sysconfig["specialtitle_0"]))
+	FilterRegTitle_1 = regexp.MustCompile(qu.ObjToString(Sysconfig["specialtitle_1"]))
+	FilterRegTitle_2 = regexp.MustCompile(qu.ObjToString(Sysconfig["specialtitle_2"]))
+	threadNum = qu.IntAllDef(Sysconfig["threads"], 1)
+	LowHeavy = Sysconfig["lowHeavy"].(bool)
+	TimingTask = Sysconfig["timingTask"].(bool)
+	timingSpanDay = qu.Int64All(Sysconfig["timingSpanDay"])
+	timingPubScope = qu.Int64All(Sysconfig["timingPubScope"])
+}
+func initOther() {
+	dupdays = qu.IntAllDef(Sysconfig["dupdays"], 5)
+	DM = NewDatamap(dupdays, lastid)
+	Update = newUpdatePool()
+	go Update.updateData()
+
+	if !IsFull {
+		var err error
+		nspdata_1, err = nsqdata.NewProducer("172.17.145.179:4150", "bidding_id", true)
+		if err != nil {
+			log.Fatal("通道配置异常~", err)
+		} else {
+			log.Println("通道配置正常")
+		}
+		nspdata_2, err = nsqdata.NewProducer("172.17.145.179:4150", "project_id", true)
+		if err != nil {
+			log.Fatal("通道配置异常~", err)
+		} else {
+			log.Println("通道配置正常~")
+		}
+	}
+	c := cron.New()
+	c.AddFunc("0 0 6 * * ?", func() {
+		isUpdateSite = true
+	})
+	c.Start()
+}
+func initSite() {
+	cronlock.Lock()
+	isUpdateSite = false
+	SiteMap = make(map[string]map[string]interface{}, 0)
+	sess := spider_mgo.GetMgoConn()
+	defer data_mgo.DestoryMongoConn(sess)
+	q := map[string]interface{}{}
+	res := sess.DB(spider_mgo.DbName).C(spider_coll).Find(&q).Sort("_id").Iter()
+	for tmp := make(map[string]interface{}); res.Next(&tmp); {
+		data := map[string]interface{}{
+			"area":     qu.ObjToString(tmp["area"]),
+			"city":     qu.ObjToString(tmp["city"]),
+			"district": qu.ObjToString(tmp["district"]),
+		}
+		SiteMap[qu.ObjToString(tmp["site"])] = data
+	}
+	log.Println("new站点加载完毕~", len(SiteMap))
+	cronlock.Unlock()
+}

src/main.go

@@ -7,7 +7,6 @@ package main
 import (
 	"encoding/json"
 	"flag"
-	"github.com/cron"
 	"log"
 	mu "mfw/util"
 	"net"
@@ -19,17 +18,16 @@ import (
 )
 
 var (
-	Sysconfig                          map[string]interface{} //配置文件
-	mconf                              map[string]interface{}
-	data_mgo, task_mgo                 *MongodbSim
-	task_collName, task_bidding        string
-	extract, extract_back, extract_log string
-	udpclient                          mu.UdpClient
-	nextNode                           []map[string]interface{}
-	dupdays                            = 7
-	DM, FullDM                         *datamap
-	Update                             *updateInfo
-	AddGroupPool                       *addGroupInfo
+	Sysconfig                            map[string]interface{} //配置文件
+	data_mgo, task_mgo, spider_mgo       *MongodbSim
+	task_coll, task_bidding, spider_coll string
+	extract, extract_back, extract_log   string
+	udpclient                            mu.UdpClient
+	nextNode                             []map[string]interface{}
+	dupdays                              = 7
+	DM, FullDM                           *datamap
+	Update                               *updateInfo
+	AddGroupPool                         *addGroupInfo
 	//正则筛选相关
 	FilterRegTitle                             = regexp.MustCompile("^_$")
 	FilterRegTitle_0                           = regexp.MustCompile("^_$")
@@ -41,106 +39,11 @@ var (
 	timingSpanDay, timingPubScope              int64
 	gtid, lastid, sec_gtid, sec_lteid, lteid   string
 	updatelock, datalock, numlock, cronlock    sync.Mutex
-	userName, passWord                         string
 	jyfb_data                                  map[string]string
 	taskList                                   []map[string]interface{}
 	nspdata_1, nspdata_2                       *nsqdata.Producer
 )
 
-func initMgo() {
-	userName = qu.ObjToString(Sysconfig["userName"])
-	passWord = qu.ObjToString(Sysconfig["passWord"])
-	log.Println("集群用户密码：", userName, passWord)
-	jyfb_arr := qu.ObjArrToStringArr(Sysconfig["jyfb_data"].([]interface{}))
-	jyfb_data = make(map[string]string, 0)
-	for _, v := range jyfb_arr {
-		jyfb_data[v] = v
-	}
-	log.Println("伪判重~", jyfb_data)
-
-	task_mconf := Sysconfig["task_mongodb"].(map[string]interface{})
-	task_mgo = &MongodbSim{
-		MongodbAddr: task_mconf["task_addrName"].(string),
-		DbName:      task_mconf["task_dbName"].(string),
-		Size:        qu.IntAllDef(task_mconf["task_pool"], 10),
-		UserName:    userName,
-		Password:    passWord,
-	}
-	task_mgo.InitPool()
-	task_collName = task_mconf["task_collName"].(string)
-	task_bidding = task_mconf["task_bidding"].(string)
-
-	nextNode = qu.ObjArrToMapArr(Sysconfig["nextNode"].([]interface{}))
-	mconf = Sysconfig["mongodb"].(map[string]interface{})
-	data_mgo = &MongodbSim{
-		MongodbAddr: mconf["addr"].(string),
-		DbName:      mconf["db"].(string),
-		Size:        qu.IntAllDef(mconf["pool"], 10),
-	}
-	data_mgo.InitPool()
-
-	extract = mconf["extract"].(string)
-	extract_back = mconf["extract_back"].(string)
-	extract_log = mconf["extract_log"].(string)
-
-	FilterRegTitle = regexp.MustCompile(qu.ObjToString(Sysconfig["specialwords"]))
-	FilterRegTitle_0 = regexp.MustCompile(qu.ObjToString(Sysconfig["specialtitle_0"]))
-	FilterRegTitle_1 = regexp.MustCompile(qu.ObjToString(Sysconfig["specialtitle_1"]))
-	FilterRegTitle_2 = regexp.MustCompile(qu.ObjToString(Sysconfig["specialtitle_2"]))
-	threadNum = qu.IntAllDef(Sysconfig["threads"], 1)
-	LowHeavy = Sysconfig["lowHeavy"].(bool)
-	TimingTask = Sysconfig["timingTask"].(bool)
-	timingSpanDay = qu.Int64All(Sysconfig["timingSpanDay"])
-	timingPubScope = qu.Int64All(Sysconfig["timingPubScope"])
-}
-func initOther() {
-	dupdays = qu.IntAllDef(Sysconfig["dupdays"], 5)
-	DM = NewDatamap(dupdays, lastid)
-	Update = newUpdatePool()
-	go Update.updateData()
-
-	if !IsFull {
-		var err error
-		nspdata_1, err = nsqdata.NewProducer("172.17.4.232:4150", "bidding_id", true)
-		if err != nil {
-			log.Fatal("通道配置异常~", err)
-		} else {
-			log.Println("通道配置正常")
-		}
-		nspdata_2, err = nsqdata.NewProducer("172.17.4.232:4150", "project_id", true)
-		if err != nil {
-			log.Fatal("通道配置异常~", err)
-		} else {
-			log.Println("通道配置正常~")
-		}
-	}
-	c := cron.New()
-	c.AddFunc("0 0 1 ? * WED", func() {
-		isUpdateSite = true
-	})
-	c.Start()
-}
-func initSite() {
-	cronlock.Lock()
-	site := mconf["site"].(map[string]interface{})
-	SiteMap = make(map[string]map[string]interface{}, 0)
-	start := int(time.Now().Unix())
-	sess_site := data_mgo.GetMgoConn()
-	defer data_mgo.DestoryMongoConn(sess_site)
-	res_site := sess_site.DB(site["dbname"].(string)).C(site["coll"].(string)).Find(map[string]interface{}{}).Sort("_id").Iter()
-	for site_dict := make(map[string]interface{}); res_site.Next(&site_dict); {
-		data_map := map[string]interface{}{
-			"area":     qu.ObjToString(site_dict["area"]),
-			"city":     qu.ObjToString(site_dict["city"]),
-			"district": qu.ObjToString(site_dict["district"]),
-		}
-		SiteMap[qu.ObjToString(site_dict["site"])] = data_map
-	}
-	isUpdateSite = false
-	log.Printf("new站点加载用时：%d秒,%d个\n", int(time.Now().Unix())-start, len(SiteMap))
-	cronlock.Unlock()
-}
-
 //初始化加载
 func init() {
 	flag.StringVar(&lastid, "id", "", "增量加载的lastid") //增量
@@ -148,7 +51,6 @@ func init() {
 	flag.StringVar(&sec_gtid, "sec_gtid", "", "全量分段起始id")
 	flag.StringVar(&sec_lteid, "sec_lteid", "", "全量分段结束id")
 	flag.Parse()
-
 	qu.ReadConfig(&Sysconfig)
 	initMgo()
 	initOther()
@@ -172,18 +74,16 @@ func mainT() {
 
 //主函数
 func main() {
-
 	go checkMapJob()
 	updport := Sysconfig["udpport"].(string)
 	udpclient = mu.UdpClient{Local: updport, BufSize: 1024}
 	udpclient.Listen(processUdpMsg)
 	log.Println("Udp服务监听", updport)
-
 	if TimingTask {
 		log.Println("正常历史部署")
 		go historyRepeat()
 	} else {
-		if !IsFull { //正常增量
+		if !IsFull {
 			log.Println("正常增量部署,监听任务")
 			go getRepeatTask()
 		}