分布式抽取

src/jy/admin/distribution/distribution.go

@@ -18,7 +18,7 @@ func init() {
 
 	//获取ecs列表
 	Admin.POST("/distribution/ecslist", func(c *gin.Context) {
-		data, _ := db.Mgo.Find("ecs", ``, `{"_id":-1}`, nil, false, -1, -1)
+		data, _ := db.Mgo.Find("ecs", `{}`, `{"_id":-1}`, nil, false, -1, -1)
 		c.JSON(200, gin.H{"data": data})
 	})
 
@@ -27,7 +27,9 @@ func init() {
 		s_table, _ := c.GetPostForm("s_table")
 		instanceid, _ := c.GetPostForm("instanceid")
 		if s_table != "" { //id划段
-			ecs.IdsRange(s_table)
+			endate, _ := c.GetPostForm("s_date")
+			log.Println(s_table, endate)
+			ecs.IdsRange(s_table, endate)
 		} else if instanceid != "" { //实例自动释放时间
 			hours, _ := c.GetPostForm("hours")
 			hour := qu.IntAll(hours)

src/jy/cluster/distributed.go

@@ -21,11 +21,12 @@ import (
 var EscIds map[string][]string //id区间
 
 //根据esc数量实例数量id划段
-func IdsRange(table string) int {
+func IdsRange(table, endate string) int {
 	start := time.Date(2015, 11, 3, 0, 0, 0, 0, time.Local)
+	end, _ := time.ParseInLocation(qu.Date_Short_Layout, endate, time.Local)
 	EscIds = map[string][]string{}
 	list, _ := db.Mgo.Find("ecs", `{"Status":"Running"}`, nil, nil, false, -1, -1)
-	ids := RangeIdsByDate(len(*list), start)
+	ids := RangeIdsByDate(len(*list), start, end)
 	for k, v := range *list {
 		db.Mgo.UpdateById("ecs", qu.BsonIdToSId(v["_id"]), map[string]interface{}{
 			"$set": map[string]interface{}{
@@ -71,12 +72,12 @@ func RunEcsTask() int {
 }
 
 //id分段
-func RangeIdsByDate(escnum int, start time.Time) map[string][]string {
+func RangeIdsByDate(escnum int, start, edate time.Time) map[string][]string {
 	ids := map[string][]string{}
 	task, _ := db.Mgo.FindById("task", qu.ObjToString(ju.Config["udptaskid"]), nil)
 	log.Println(qu.ObjToString((*task)["s_mgoaddr"]), qu.ObjToString((*task)["s_mgodb"]))
 	DB := db.MgoFactory(2, 3, 120, qu.ObjToString((*task)["s_mgoaddr"]), qu.ObjToString((*task)["s_mgodb"]))
-	total := DB.Count("bidding", `{}`)
+	total := DB.Count("bidding", `{"comeintime":{"$lt":`+fmt.Sprint(edate.Unix())+`}}`)
 	total_back := DB.Count("bidding_back", `{}`)
 	total += total_back
 	pagesize := (total + escnum - 1) / escnum
@@ -90,14 +91,18 @@ func RangeIdsByDate(escnum int, start time.Time) map[string][]string {
 		table := "bidding_back"
 		for {
 			tmpsid := bson.NewObjectIdWithTime(start)
-			end := start.Add(12 * time.Hour)
-			eid = bson.NewObjectIdWithTime(end)
+			end := start.Add(4 * time.Hour)
+			if end.Unix() > edate.Unix() {
+				eid = bson.NewObjectIdWithTime(edate)
+			} else {
+				eid = bson.NewObjectIdWithTime(end)
+			}
 			start = end
 			query := bson.M{"_id": bson.M{"$gte": tmpsid, "$lt": eid}}
 			count := DB.Count(table, query)
 			log.Println(count, table, query)
 			if count < 1 { //校验是否切换table
-				tmpnum := DB.Count(table, bson.M{"_id": bson.M{"$gte": tmpsid, "$lt": bson.NewObjectIdWithTime(end.Add(24 * 10 * time.Hour))}})
+				tmpnum := DB.Count(table, bson.M{"_id": bson.M{"$gte": tmpsid, "$lt": bson.NewObjectIdWithTime(end.Add(24 * 10 * time.Hour) /*连续10天无数据*/)}})
 				if tmpnum < 1 && table != "bidding" {
 					table = "bidding"
 					start = start.Add(-12 * time.Hour)

src/res/fieldscore.json

@@ -24,13 +24,13 @@
         ],
         "length": [
             {
-                "describe": "长度打分min>val:0,min<=val<=max:1,max<val:-1",
+                "describe": "长度打分min>val:-6,min<=val<=max:1,max<val:-1",
                 "min": 4,
                 "max": 35,
                 "score": [
-                    0,
+                    -6,
                     1,
-                    -5
+                    -1
                 ]
             }
         ]
@@ -40,7 +40,7 @@
         "position": [
             {
                 "describe": "以*结尾",
-                "regstr": ".{2,100}(委员会|办公室|局|中心|协会|公司|政府|大学|学校|医院|集团|银行)$",
+                "regstr": ".{2,100}(处|委员会|办公室|局|中心|协会|公司|政府|大学|学校|医院|集团|银行)$",
                 "score": 3
             },
             {
@@ -51,13 +51,13 @@
         ],
         "length": [
             {
-                "describe": "长度打分min>val:0,min<=val<=max:1,max<val:-1",
+                "describe": "长度打分min>val:-6,min<=val<=max:1,max<val:-1",
                 "min": 4,
                 "max": 15,
                 "score": [
-                    0,
+                    -6,
                     1,
-                    -5
+                    -1
                 ]
             }
         ]
@@ -73,13 +73,13 @@
         ],
         "length": [
             {
-                "describe": "长度打分min>val:0,min<=val<=max:1,max<val:-1",
+                "describe": "长度打分min>val:-6,min<=val<=max:1,max<val:-1",
                 "min": 4,
                 "max": 15,
                 "score": [
-                    0,
+                    -6,
                     1,
-                    -5
+                    -1
                 ]
             }
         ]
@@ -100,13 +100,13 @@
         ],
         "length": [
             {
-                "describe": "长度打分min>val:0,min<=val<=max:1,max<val:-1",
+                "describe": "长度打分min>val:-6,min<=val<=max:1,max<val:-1",
                 "min": 4,
                 "max": 15,
                 "score": [
-                    0,
+                    -6,
                     1,
-                    -5
+                    -1
                 ]
             }
         ]

src/web/templates/admin/distribution.html

@@ -132,6 +132,7 @@ $(function () {
                   _tit="id划段"
     				tag=[
     					{label:"表名",s_label:"s_table",placeholder:"信息表名",must:true},
+                        {label:"截止日期",s_label:"s_date",placeholder:"2019-01-01",must:true},
     				]   
                 }else if (n=="releasetime"){
                     _tit="释放设置"