|
@@ -15,11 +15,13 @@ import (
|
|
|
var N = 200
|
|
|
|
|
|
//生enterprise表
|
|
|
-func main() {
|
|
|
+func main1() {
|
|
|
// 参数 mongodb(网站库) mongodb(公示库) elastic collection area index-db inde-type N
|
|
|
-
|
|
|
initConfig(os.Args[1], "web", "qfw")
|
|
|
- initConfig(os.Args[2], "ecps", "qfw")
|
|
|
+ if os.Args[4] == "enterprise" {
|
|
|
+ initConfig(os.Args[2], "ecps", "qfw")
|
|
|
+
|
|
|
+ }
|
|
|
InitMongodbPool()
|
|
|
elastic.InitElastic(os.Args[3])
|
|
|
if len(os.Args[8]) > 0 {
|
|
@@ -40,7 +42,7 @@ func main() {
|
|
|
query := ecps_session.DB("qfw").C(os.Args[4]).Find(m).Iter()
|
|
|
arr := make([]map[string]interface{}, 0)
|
|
|
for tmp := make(map[string]interface{}); query.Next(tmp); i++ {
|
|
|
- arr = append(arr, elastic.ConverData(&tmp))
|
|
|
+ arr = append(arr, tmp)
|
|
|
if i%N == 0 && i != 0 {
|
|
|
elastic.BulkSave(os.Args[6], os.Args[7], &arr, true)
|
|
|
arr = make([]map[string]interface{}, 0)
|
|
@@ -51,53 +53,83 @@ func main() {
|
|
|
elastic.BulkSave(os.Args[6], os.Args[7], &arr, true)
|
|
|
arr = make([]map[string]interface{}, 0)
|
|
|
}
|
|
|
-
|
|
|
- //生成带名片索引
|
|
|
- i = 0
|
|
|
- count := Count("enterprise", "web", "qfw", m)
|
|
|
- log.Println("网站库,一共", count, "条数据")
|
|
|
- for {
|
|
|
- limit := i + N
|
|
|
- if limit > count {
|
|
|
- limit = count
|
|
|
- }
|
|
|
- query := *Find("enterprise", "web", "qfw", m, nil, nil, false, i, i+N)
|
|
|
- regNos := make([]string, 0)
|
|
|
- arr := make([]map[string]interface{}, 0)
|
|
|
- if len(query) > 0 {
|
|
|
- for _, v := range query {
|
|
|
- if v["_id"] != nil {
|
|
|
- regno := fmt.Sprint(v["RegNo"])
|
|
|
- regNos = append(regNos, "\""+regno+"\"")
|
|
|
- delete(v, "_id")
|
|
|
- }
|
|
|
+ if os.Args[4] == "enterprise" {
|
|
|
+ //生成带名片索引
|
|
|
+ i = 0
|
|
|
+ count := Count("enterprise", "web", "qfw", m)
|
|
|
+ log.Println("网站库,一共", count, "条数据")
|
|
|
+ for {
|
|
|
+ limit := i + N
|
|
|
+ if limit > count {
|
|
|
+ limit = count
|
|
|
}
|
|
|
-
|
|
|
- findstr := `{"RegNo":{"$in":[` + strings.Join(regNos, ",") + `]}}`
|
|
|
- query1 := *Find("enterprise", "ecps", "qfw", findstr, nil, nil, false, -1, -1)
|
|
|
- for _, v := range query {
|
|
|
- for _, _v := range query1 {
|
|
|
- if fmt.Sprint(_v["RegNo"]) == fmt.Sprint(v["RegNo"]) {
|
|
|
+ query := *Find("enterprise", "web", "qfw", m, nil, nil, false, i, i+N)
|
|
|
+ regNos := make([]string, 0)
|
|
|
+ arr := make([]map[string]interface{}, 0)
|
|
|
+ if len(query) > 0 {
|
|
|
+ for _, v := range query {
|
|
|
+ if v["_id"] != nil {
|
|
|
+ regno := fmt.Sprint(v["RegNo"])
|
|
|
+ regNos = append(regNos, "\""+regno+"\"")
|
|
|
delete(v, "_id")
|
|
|
- if len(v) > 1 {
|
|
|
- for field, value := range v {
|
|
|
- _v[field] = value
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ findstr := `{"RegNo":{"$in":[` + strings.Join(regNos, ",") + `]}}`
|
|
|
+ query1 := *Find("enterprise", "ecps", "qfw", findstr, nil, nil, false, -1, -1)
|
|
|
+ for _, v := range query {
|
|
|
+ for _, _v := range query1 {
|
|
|
+ if fmt.Sprint(_v["RegNo"]) == fmt.Sprint(v["RegNo"]) {
|
|
|
+ delete(v, "_id")
|
|
|
+ if len(v) > 1 {
|
|
|
+ for field, value := range v {
|
|
|
+ _v[field] = value
|
|
|
+ }
|
|
|
+ arr = append(arr, elastic.ConverData(&_v))
|
|
|
}
|
|
|
- arr = append(arr, elastic.ConverData(&_v))
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
+ elastic.BulkSave(os.Args[6], os.Args[7], &arr, true)
|
|
|
}
|
|
|
- elastic.BulkSave(os.Args[6], os.Args[7], &arr, true)
|
|
|
+ if i >= count {
|
|
|
+ break
|
|
|
+ } else {
|
|
|
+ i = i + N
|
|
|
+ }
|
|
|
+ log.Printf("当前索引位置:=%d \n", i)
|
|
|
}
|
|
|
- if i >= count {
|
|
|
- break
|
|
|
+ }
|
|
|
+
|
|
|
+}
|
|
|
+func main() {
|
|
|
+ // 参数 mongodb(网站库) elastic collection query
|
|
|
+ initConfig(os.Args[1], "web", "qfw")
|
|
|
+ InitMongodbPool()
|
|
|
+ log.Println("初始化完毕")
|
|
|
+ elastic.InitElastic(os.Args[2])
|
|
|
+ log.Println("elatic初始化完毕")
|
|
|
+ query := ""
|
|
|
+ if len(os.Args) > 4 && os.Args[4] != "" {
|
|
|
+ query = os.Args[4]
|
|
|
+ query = strings.Replace(query, "^", "$", -1)
|
|
|
+ }
|
|
|
+ log.Println("query:", query)
|
|
|
+ list := *Find(os.Args[3], "web", "qfw", query, nil, nil, false, -1, -1)
|
|
|
+ log.Println("len:", len(list))
|
|
|
+ log.Println("查询完毕")
|
|
|
+ var i = 0
|
|
|
+ for i <= len(list) {
|
|
|
+ var tmp []map[string]interface{}
|
|
|
+ if i+100 >= len(list) {
|
|
|
+ tmp = list[i:len(list)]
|
|
|
} else {
|
|
|
- i = i + N
|
|
|
+ tmp = list[i : i+100]
|
|
|
}
|
|
|
- log.Printf("当前索引位置:=%d \n", i)
|
|
|
+ elastic.BulkSave(os.Args[3], os.Args[3], &tmp, true)
|
|
|
+ log.Println("当前索引:", i)
|
|
|
+ i = i + 100
|
|
|
}
|
|
|
-
|
|
|
}
|
|
|
func initConfig(ip, alias, db string) {
|
|
|
conf := *new(PoolConfig)
|