|
|
@@ -1,6 +1,8 @@
|
|
|
package main
|
|
|
|
|
|
import (
|
|
|
+ . "app.yhyue.com/moapp/jybase/mongodb"
|
|
|
+ "flag"
|
|
|
"github.com/gogf/gf/v2/frame/g"
|
|
|
"github.com/gogf/gf/v2/os/gctx"
|
|
|
"github.com/gogf/gf/v2/os/gfsnotify"
|
|
|
@@ -8,12 +10,15 @@ import (
|
|
|
"gopkg.in/natefinch/lumberjack.v2"
|
|
|
"jygit.jydev.jianyu360.cn/BaseService/ossClient/constant"
|
|
|
"jygit.jydev.jianyu360.cn/BaseService/ossClient/pb"
|
|
|
+ "jygit.jydev.jianyu360.cn/BaseService/ossService/config"
|
|
|
ossService "jygit.jydev.jianyu360.cn/BaseService/ossService/oss"
|
|
|
"jygit.jydev.jianyu360.cn/BaseService/ossService/util"
|
|
|
"log"
|
|
|
"net"
|
|
|
"net/http"
|
|
|
"net/rpc"
|
|
|
+ "strings"
|
|
|
+ "sync"
|
|
|
"time"
|
|
|
)
|
|
|
|
|
|
@@ -92,3 +97,76 @@ func main() {
|
|
|
log.Fatalln("HTTP server error", err)
|
|
|
}
|
|
|
}
|
|
|
+
|
|
|
+// /////////////////
|
|
|
+func main11() {
|
|
|
+ start := flag.Int64("s", 0, "")
|
|
|
+ end := flag.Int64("e", 0, "")
|
|
|
+ poolSize := flag.Int("p", 5, "")
|
|
|
+ flag.Parse()
|
|
|
+ // 初始化OSS帐号与bucket信息
|
|
|
+ ossService.LoadOSSAccounts()
|
|
|
+ sess := config.Mgo.GetMgoConn()
|
|
|
+ defer config.Mgo.DestoryMongoConn(sess)
|
|
|
+ query := map[string]interface{}{}
|
|
|
+ idQuery := map[string]interface{}{}
|
|
|
+ if *start != 0 && *start == *end {
|
|
|
+ query["comeintime"] = *start
|
|
|
+ } else {
|
|
|
+ if *start != 0 {
|
|
|
+ idQuery["$gte"] = *start
|
|
|
+ }
|
|
|
+ if *end != 0 {
|
|
|
+ idQuery["$lte"] = *end
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if len(idQuery) > 0 {
|
|
|
+ query["comeintime"] = idQuery
|
|
|
+ }
|
|
|
+ log.Println("start。。。", query)
|
|
|
+ it := sess.DB(config.Mgo.DbName).C(g.Config().MustGet(gctx.New(), "mongodb.collection").String()).Find(query).Select(map[string]interface{}{
|
|
|
+ "_id": 1,
|
|
|
+ "detail": 1,
|
|
|
+ "contenthtml": 1,
|
|
|
+ "comeintime": 1,
|
|
|
+ }).Sort("-comeintime").Iter()
|
|
|
+ pool := make(chan bool, *poolSize)
|
|
|
+ wait := &sync.WaitGroup{}
|
|
|
+ index := 0
|
|
|
+ for m := make(map[string]interface{}); it.Next(&m); {
|
|
|
+ pool <- true
|
|
|
+ wait.Add(1)
|
|
|
+ index++
|
|
|
+ if index%5000 == 0 {
|
|
|
+ log.Println(index, m["_id"], m["comeintime"])
|
|
|
+ }
|
|
|
+ go func(mm map[string]interface{}) {
|
|
|
+ defer func() {
|
|
|
+ <-pool
|
|
|
+ wait.Done()
|
|
|
+ }()
|
|
|
+ objectName := BsonIdToSId(mm["_id"]) + ".txt"
|
|
|
+ detail, _ := mm["detail"].(string)
|
|
|
+ for {
|
|
|
+ if err1 := ossService.Upload("detail", objectName, strings.NewReader(detail), false); err1 != nil {
|
|
|
+ log.Println("detail", objectName, err1)
|
|
|
+ time.Sleep(3 * time.Second)
|
|
|
+ } else {
|
|
|
+ break
|
|
|
+ }
|
|
|
+ }
|
|
|
+ contenthtml, _ := mm["contenthtml"].(string)
|
|
|
+ for {
|
|
|
+ if err2 := ossService.Upload("contenthtml", objectName, strings.NewReader(contenthtml), false); err2 != nil {
|
|
|
+ log.Println("contenthtml", objectName, err2)
|
|
|
+ time.Sleep(3 * time.Second)
|
|
|
+ } else {
|
|
|
+ break
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }(m)
|
|
|
+ m = make(map[string]interface{})
|
|
|
+ }
|
|
|
+ wait.Wait()
|
|
|
+ log.Println("over。。。", index)
|
|
|
+}
|
