|
|
@@ -6,6 +6,7 @@ package main
|
|
|
|
|
|
import (
|
|
|
"encoding/json"
|
|
|
+ "flag"
|
|
|
"fmt"
|
|
|
"gopkg.in/mgo.v2/bson"
|
|
|
"log"
|
|
|
@@ -30,7 +31,7 @@ var (
|
|
|
dupdays = 5 //初始化判重范围
|
|
|
DM *datamap //判重数据
|
|
|
FilterRegexp = regexp.MustCompile("^_$")
|
|
|
- lastid = "5d767728a5cb26b9b7748868" //起始数据1条
|
|
|
+ lastid = "" //起始数据1条
|
|
|
|
|
|
|
|
|
FilterRegTitle = regexp.MustCompile("[0-9一二三四五六七八九十零１２３４５６７８９](次|包|标段)")
|
|
|
@@ -42,9 +43,9 @@ var (
|
|
|
)
|
|
|
|
|
|
func init() {
|
|
|
- //flag.StringVar(&lastid, "id", "", "最后加载id") //以小于等于此id开始加载最近几天的数据
|
|
|
- //flag.Parse()// 类似于命令台 scanf
|
|
|
- ////172.17.145.163:27080
|
|
|
+ flag.StringVar(&lastid, "id", "", "最后加载id") //以小于等于此id开始加载最近几天的数据
|
|
|
+ flag.Parse()// 类似于命令台 scanf
|
|
|
+ //172.17.145.163:27080
|
|
|
util.ReadConfig(&Sysconfig)
|
|
|
nextNode = util.ObjArrToMapArr(Sysconfig["nextNode"].([]interface{}))
|
|
|
mconf = Sysconfig["mongodb"].(map[string]interface{})
|
|
|
@@ -72,7 +73,7 @@ func init() {
|
|
|
}
|
|
|
|
|
|
//新增一个方法 判断
|
|
|
-func main22() {
|
|
|
+func mainTest() {
|
|
|
|
|
|
sess := mgo.GetMgoConn()
|
|
|
defer mgo.DestoryMongoConn(sess)
|
|
|
@@ -243,13 +244,7 @@ func task(data []byte, mapInfo map[string]interface{}) {
|
|
|
b, id := DM.check(info)
|
|
|
if b { //有重复,生成更新语句，更新抽取和更新招标
|
|
|
|
|
|
- //重复数据，处理方式 合并操作--------------
|
|
|
- //重复数据，处理方式 合并操作--------------
|
|
|
- //重复数据，处理方式 合并操作--------------
|
|
|
-
|
|
|
-
|
|
|
repeateN++
|
|
|
- //log.Println("变更数据重复",tmp["_id"])
|
|
|
mapLock.Lock()
|
|
|
updateExtract = append(updateExtract, []map[string]interface{}{
|
|
|
map[string]interface{}{
|
