|
@@ -43,76 +43,80 @@ func main() {
|
|
|
if isEnterprise {
|
|
|
dbType = "ecps"
|
|
|
}
|
|
|
- count := Count(os.Args[4], dbType, "qfw", dbType)
|
|
|
- totalPages := ((count + N - 1) / N)
|
|
|
- log.Println(util.If(isEnterprise, "公式库", "网站库"), os.Args[4], "表有", count, "条数据", "一共", totalPages, "页")
|
|
|
+ count := Count(os.Args[4], dbType, "qfw", m)
|
|
|
+ log.Println(util.If(isEnterprise, "公式库", "网站库"), os.Args[4], "表有", count, "条数据")
|
|
|
sess := GetMgoConn(dbType)
|
|
|
defer DestoryMongoConn(dbType, sess)
|
|
|
query := sess.DB("qfw").C(os.Args[4]).Find(m).Iter()
|
|
|
arr := make([]map[string]interface{}, 0)
|
|
|
i := 0
|
|
|
for tmp := make(map[string]interface{}); query.Next(tmp); i++ {
|
|
|
- if i >= 12690000 {
|
|
|
- arr = append(arr, elastic.ConverData(&tmp))
|
|
|
- if len(arr) == N {
|
|
|
- elastic.BulkSave(os.Args[6], os.Args[7], &arr, true)
|
|
|
- arr = make([]map[string]interface{}, 0)
|
|
|
- }
|
|
|
+ arr = append(arr, elastic.ConverData(&tmp))
|
|
|
+ if len(arr) == N {
|
|
|
+ elastic.BulkSave(os.Args[6], os.Args[7], &arr, true)
|
|
|
+ arr = make([]map[string]interface{}, 0)
|
|
|
}
|
|
|
if i%N == 0 {
|
|
|
log.Printf("当前索引位置:=%d \n", i)
|
|
|
}
|
|
|
}
|
|
|
- log.Println("剩余.....", len(arr))
|
|
|
if len(arr) > 0 {
|
|
|
+ log.Println("剩余.....", len(arr))
|
|
|
elastic.BulkSave(os.Args[6], os.Args[7], &arr, true)
|
|
|
arr = make([]map[string]interface{}, 0)
|
|
|
}
|
|
|
if isEnterprise {
|
|
|
- //生成带名片索引
|
|
|
i = 0
|
|
|
+ s := 0
|
|
|
+ //生成带名片索引
|
|
|
count := Count("enterprise", "web", "qfw", m)
|
|
|
log.Println("网站库enterprise表,一共", count, "条数据")
|
|
|
- for {
|
|
|
- limit := i + N
|
|
|
- if limit > count {
|
|
|
- limit = count
|
|
|
+ query := *Find("enterprise", "web", "qfw", m, nil, nil, false, -1, -1)
|
|
|
+ regNos := make([]string, 0)
|
|
|
+ arr := make([]map[string]interface{}, 0)
|
|
|
+ if len(query) > 0 {
|
|
|
+ for _, v := range query {
|
|
|
+ if v["_id"] != nil {
|
|
|
+ regno := fmt.Sprint(v["RegNo"])
|
|
|
+ regNos = append(regNos, "\""+regno+"\"")
|
|
|
+ delete(v, "_id")
|
|
|
+ }
|
|
|
+ }
|
|
|
+ findstr := `{"RegNo":{"$in":[` + strings.Join(regNos, ",") + `]}}`
|
|
|
+ query1 := *Find("enterprise", "ecps", "qfw", findstr, nil, nil, false, -1, -1)
|
|
|
+ query1Map := make(map[string]interface{})
|
|
|
+ for _, _v := range query1 {
|
|
|
+ query1Map[fmt.Sprint(_v["RegNo"])] = _v
|
|
|
}
|
|
|
- query := *Find("enterprise", "web", "qfw", m, nil, nil, false, i, i+N)
|
|
|
- regNos := make([]string, 0)
|
|
|
- arr := make([]map[string]interface{}, 0)
|
|
|
- if len(query) > 0 {
|
|
|
- for _, v := range query {
|
|
|
- if v["_id"] != nil {
|
|
|
- regno := fmt.Sprint(v["RegNo"])
|
|
|
- regNos = append(regNos, "\""+regno+"\"")
|
|
|
- delete(v, "_id")
|
|
|
+ for _, v := range query {
|
|
|
+ if len(v) > 1 {
|
|
|
+ i++
|
|
|
+ _v, _ := query1Map[fmt.Sprint(v["RegNo"])].(map[string]interface{})
|
|
|
+ if _v == nil || len(_v) == 0 {
|
|
|
+ s++
|
|
|
+ continue
|
|
|
}
|
|
|
- }
|
|
|
-
|
|
|
- findstr := `{"RegNo":{"$in":[` + strings.Join(regNos, ",") + `]}}`
|
|
|
- query1 := *Find("enterprise", "ecps", "qfw", findstr, nil, nil, false, -1, -1)
|
|
|
- for _, v := range query {
|
|
|
- for _, _v := range query1 {
|
|
|
- if fmt.Sprint(_v["RegNo"]) == fmt.Sprint(v["RegNo"]) {
|
|
|
- delete(v, "_id")
|
|
|
- if len(v) > 1 {
|
|
|
- for field, value := range v {
|
|
|
- _v[field] = value
|
|
|
- }
|
|
|
- arr = append(arr, elastic.ConverData(&_v))
|
|
|
- }
|
|
|
- }
|
|
|
+ for field, value := range v {
|
|
|
+ _v[field] = value
|
|
|
+ }
|
|
|
+ arr = append(arr, elastic.ConverData(&_v))
|
|
|
+ if len(arr) == N {
|
|
|
+ elastic.BulkSave(os.Args[6], os.Args[7], &arr, true)
|
|
|
+ arr = make([]map[string]interface{}, 0)
|
|
|
+ }
|
|
|
+ if i%N == 0 {
|
|
|
+ log.Printf("当前索引位置:=%d \n", i)
|
|
|
}
|
|
|
}
|
|
|
+ }
|
|
|
+ if len(arr) > 0 {
|
|
|
+ log.Println("剩余.....", len(arr))
|
|
|
elastic.BulkSave(os.Args[6], os.Args[7], &arr, true)
|
|
|
+ arr = make([]map[string]interface{}, 0)
|
|
|
}
|
|
|
- if i >= count {
|
|
|
- break
|
|
|
- } else {
|
|
|
- i = i + N
|
|
|
+ if s > 0 {
|
|
|
+ log.Println("网站库有", s, "条在公式库中找不到")
|
|
|
}
|
|
|
- log.Printf("当前索引位置:=%d \n", i)
|
|
|
}
|
|
|
}
|
|
|
log.Println("-------------结束-----------------")
|