|
|
@@ -32,9 +32,10 @@ var (
|
|
|
dupdays = 5 //初始化判重范围
|
|
|
DM *datamap //判重数据
|
|
|
FilterRegexp = regexp.MustCompile("^_$")
|
|
|
- lastid = "5d767728a5cb26b9b7748868"
|
|
|
-
|
|
|
+ lastid = "5da3f2c5a5cb26b9b79847fc"
|
|
|
|
|
|
+ //5d767728a5cb26b9b7748868 //9万
|
|
|
+ //5da3f2c5a5cb26b9b79847fc //76万
|
|
|
FilterRegTitle = regexp.MustCompile("[0-9一二三四五六七八九十零１２３４５６７８９](次|包|标段)")
|
|
|
inV_n int //无效数据数量
|
|
|
siteArr []map[string]interface{} //站点
|
|
|
@@ -64,13 +65,13 @@ func init() {
|
|
|
|
|
|
|
|
|
//测试临时注释
|
|
|
- //dupdays = util.IntAllDef(Sysconfig["dupdays"], 3)
|
|
|
- ////加载数据
|
|
|
- //DM = NewDatamap(dupdays, lastid)
|
|
|
- //sw := util.ObjToString(Sysconfig["specialwords"])
|
|
|
- //if sw != "" {
|
|
|
- // FilterRegexp = regexp.MustCompile(sw)
|
|
|
- //}
|
|
|
+ dupdays = util.IntAllDef(Sysconfig["dupdays"], 3)
|
|
|
+ //加载数据
|
|
|
+ DM = NewDatamap(dupdays, lastid)
|
|
|
+ sw := util.ObjToString(Sysconfig["specialwords"])
|
|
|
+ if sw != "" {
|
|
|
+ FilterRegexp = regexp.MustCompile(sw)
|
|
|
+ }
|
|
|
|
|
|
|
|
|
//数据库
|
|
|
@@ -87,38 +88,38 @@ func init() {
|
|
|
}
|
|
|
|
|
|
//新增一个方法 判断
|
|
|
-func main222() {
|
|
|
- log.Println("1")
|
|
|
-
|
|
|
- sessTest :=mgoTest.GetMgoConn()
|
|
|
- defer sessTest.Close()
|
|
|
-
|
|
|
- sess := mgo.GetMgoConn()
|
|
|
- defer sess.Close()
|
|
|
-
|
|
|
- //var arr []map[string]interface{}
|
|
|
+func main22() {
|
|
|
+ //log.Println("1")
|
|
|
|
|
|
- res_test := sessTest.DB("qfw").C("bidding").Find(mongodb.ObjToMQ(`{"comeintime":{"$gte": 1571025600, "$lte": 1571976000}}`, true)).Iter()
|
|
|
- res :=sess.DB("extract_kf").C("a_testbidding")
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
- i:=0
|
|
|
- for dict := make(map[string]interface{}); res_test.Next(&dict); i++{
|
|
|
-
|
|
|
- //插入
|
|
|
- if i%2000==0 {
|
|
|
- log.Println("当前：",i)
|
|
|
- }
|
|
|
- res.Insert(dict)
|
|
|
- //if len(arr)>=500 {
|
|
|
- // arr = make([]map[string]interface{},0)
|
|
|
- //}else {
|
|
|
- // arr = append(arr,dict)
|
|
|
- //}
|
|
|
- }
|
|
|
+ //sessTest :=mgoTest.GetMgoConn()
|
|
|
+ //defer sessTest.Close()
|
|
|
+ //
|
|
|
+ //sess := mgo.GetMgoConn()
|
|
|
+ //defer sess.Close()
|
|
|
+ //
|
|
|
+ ////var arr []map[string]interface{}
|
|
|
+ //
|
|
|
+ //res_test := sessTest.DB("qfw").C("bidding").Find(mongodb.ObjToMQ(`{"comeintime":{"$gte": 1571025600, "$lte": 1571976000}}`, true)).Iter()
|
|
|
+ //res :=sess.DB("extract_kf").C("a_testbidding")
|
|
|
+ //
|
|
|
+ //
|
|
|
+ //
|
|
|
+ //
|
|
|
+ //
|
|
|
+ //i:=0
|
|
|
+ //for dict := make(map[string]interface{}); res_test.Next(&dict); i++{
|
|
|
+ //
|
|
|
+ // //插入
|
|
|
+ // if i%2000==0 {
|
|
|
+ // log.Println("当前：",i)
|
|
|
+ // }
|
|
|
+ // res.Insert(dict)
|
|
|
+ // //if len(arr)>=500 {
|
|
|
+ // // arr = make([]map[string]interface{},0)
|
|
|
+ // //}else {
|
|
|
+ // // arr = append(arr,dict)
|
|
|
+ // //}
|
|
|
+ //}
|
|
|
|
|
|
|
|
|
|
