|
|
@@ -1,53 +1,108 @@
|
|
|
package main
|
|
|
|
|
|
import (
|
|
|
+ "fmt"
|
|
|
"gopkg.in/mgo.v2/bson"
|
|
|
"strconv"
|
|
|
+ "strings"
|
|
|
|
|
|
"log"
|
|
|
"os"
|
|
|
"qfw/util/elastic"
|
|
|
- "qfw/util/mongodb"
|
|
|
+ . "qfw/util/mongodbutil"
|
|
|
)
|
|
|
|
|
|
var N = 200
|
|
|
|
|
|
//生enterprise表
|
|
|
func main() {
|
|
|
- //6 参数 mongodb elastic collection area index-db inde-type N
|
|
|
- if len(os.Args[7]) > 0 {
|
|
|
- n1, _ := strconv.Atoi(os.Args[7])
|
|
|
+ // 参数 mongodb(网站库) mongodb(公示库) elastic collection area index-db inde-type N
|
|
|
+ initConfig(os.Args[1], "web", "qfw")
|
|
|
+ initConfig(os.Args[2], "ecps", "qfw")
|
|
|
+ InitMongodbPool()
|
|
|
+ elastic.InitElastic(os.Args[3])
|
|
|
+ if len(os.Args[8]) > 0 {
|
|
|
+ n1, _ := strconv.Atoi(os.Args[8])
|
|
|
if n1 > 0 {
|
|
|
N = n1
|
|
|
}
|
|
|
}
|
|
|
- mongodb.InitMongodbPool(1, os.Args[1], "qfw")
|
|
|
- elastic.InitElastic(os.Args[2])
|
|
|
- session := mongodb.GetMgoConn()
|
|
|
+ // ecps_session := GetMgoConn("ecps")
|
|
|
var m map[string]interface{}
|
|
|
- if os.Args[4] == "all" {
|
|
|
+ if os.Args[5] == "all" {
|
|
|
m = nil
|
|
|
} else {
|
|
|
- m = bson.M{"Area": os.Args[4]}
|
|
|
+ m = bson.M{"Area": os.Args[5]}
|
|
|
}
|
|
|
log.Printf("%v", m)
|
|
|
- query := session.DB("qfw").C(os.Args[3]).Find(m).Iter()
|
|
|
- arr := make([]map[string]interface{}, N)
|
|
|
- var n int
|
|
|
i := 0
|
|
|
- for tmp := make(map[string]interface{}); query.Next(tmp); i = i + 1 {
|
|
|
- n++
|
|
|
- arr[i] = elastic.ConverData(&tmp)
|
|
|
- if i == N-1 {
|
|
|
- elastic.BulkSave(os.Args[5], os.Args[6], &arr, true)
|
|
|
- i = 0
|
|
|
- arr = make([]map[string]interface{}, N)
|
|
|
+ // query := ecps_session.DB("qfw").C(os.Args[4]).Find(m).Iter()
|
|
|
+ // arr := make([]map[string]interface{}, 0)
|
|
|
+ // for tmp := make(map[string]interface{}); query.Next(tmp); i++ {
|
|
|
+ // arr = append(arr, elastic.ConverData(&tmp))
|
|
|
+ // if i%N == 0 && i != 0 {
|
|
|
+ // elastic.BulkSave(os.Args[6], os.Args[7], &arr, true)
|
|
|
+ // arr = make([]map[string]interface{}, 0)
|
|
|
+ // log.Printf("当前索引位置:=%d \n", i)
|
|
|
+ // }
|
|
|
+ // }
|
|
|
+ // if len(arr) > 0 {
|
|
|
+ // elastic.BulkSave(os.Args[6], os.Args[7], &arr, true)
|
|
|
+ // arr = make([]map[string]interface{}, 0)
|
|
|
+ // }
|
|
|
+
|
|
|
+ //生成带名片索引
|
|
|
+ i = 0
|
|
|
+ count := Count("enterprise", "web", "qfw", m)
|
|
|
+ log.Println("网站库,一共", count, "条数据")
|
|
|
+ for {
|
|
|
+ limit := i + N
|
|
|
+ if limit > count {
|
|
|
+ limit = count
|
|
|
}
|
|
|
- if n%2000 == 0 {
|
|
|
- log.Printf("current index:=%d \n", n)
|
|
|
+ query := *Find("enterprise", "web", "qfw", m, nil, nil, false, i, i+N)
|
|
|
+ regNos := make([]string, 0)
|
|
|
+ arr := make([]map[string]interface{}, 0)
|
|
|
+ if len(query) > 0 {
|
|
|
+ for _, v := range query {
|
|
|
+ if v["_id"] != nil {
|
|
|
+ regno := fmt.Sprint(v["RegNo"])
|
|
|
+ regNos = append(regNos, "\""+regno+"\"")
|
|
|
+ delete(v, "_id")
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ findstr := `{"RegNo":{"$in":[` + strings.Join(regNos, ",") + `]}}`
|
|
|
+ query1 := *Find("enterprise", "ecps", "qfw", findstr, nil, nil, false, -1, -1)
|
|
|
+ for _, v := range query {
|
|
|
+ for _, _v := range query1 {
|
|
|
+ if fmt.Sprint(_v["RegNo"]) == fmt.Sprint(v["RegNo"]) {
|
|
|
+ delete(v, "_id")
|
|
|
+ if len(v) > 1 {
|
|
|
+ for field, value := range v {
|
|
|
+ _v[field] = value
|
|
|
+ }
|
|
|
+ arr = append(arr, elastic.ConverData(&_v))
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ elastic.BulkSave(os.Args[6], os.Args[7], &arr, true)
|
|
|
}
|
|
|
+ if i >= count {
|
|
|
+ break
|
|
|
+ } else {
|
|
|
+ i = i + N
|
|
|
+ }
|
|
|
+ log.Printf("当前索引位置:=%d \n", i)
|
|
|
}
|
|
|
- if i > 0 {
|
|
|
- elastic.BulkSave(os.Args[5], os.Args[6], &arr, true)
|
|
|
- }
|
|
|
+
|
|
|
+}
|
|
|
+func initConfig(ip, alias, db string) {
|
|
|
+ conf := *new(PoolConfig)
|
|
|
+ conf.Addr = ip
|
|
|
+ conf.Alias = alias
|
|
|
+ conf.DB = db
|
|
|
+ conf.Size = 5
|
|
|
+ Config = append(Config, conf)
|
|
|
}
|
李广朋