update

website_columns/config.toml

@@ -1,6 +1,5 @@
 [mongob]  ## bidding 链接
-#    host = "127.0.0.1:27083"
-#    #    host = "192.168.3.207:29099"
+#    host = "172.17.145.163:27083"
 #    db = "qfw"
 #    coll = "bidding"        ## 标讯数据标
 #    username = "SJZY_RWbid_ES"
@@ -15,8 +14,7 @@
 
 
 [mongop] ##  项目信息
-#    host = "127.0.0.1:27080"
-#    #    host = "192.168.3.207:29099"
+#    host = "172.17.4.85:27080"
 #    db = "qfw"
 #    coll = "projectset_proposed"    ## 拟建项目表
 #    username = ""
@@ -32,8 +30,8 @@
 
 [cron] ## 定时任务
 #    spec = "0 */1 * * * *"    ## 5分钟执行一次
-    spec = "0 00 20 * * *"     ## 每天20点执行
-    start = 1536831167         ## 表示 comeintime 大于这个值
+    spec = "0 00 01 * * *"     ## 每天01点执行
+#    start = 1536831167         ## 表示 comeintime 大于这个值
 #    end = 0                    ## 表示comeintime 小于这个值
 #    delete = -30              ## 删除30天之前的数据
     columns = ["招标公告","招标预告","招标结果","招标信用","采购意向","项目分包","企业直采",

website_columns/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"github.com/robfig/cron/v3"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo/options"
 	"go.uber.org/zap"
@@ -22,22 +23,26 @@ var (
 
 func main() {
 	go updateMethod()
-	go dealBidding()
-	//go dealProject()
-	//local, _ := time.LoadLocation("Asia/Shanghai")
-	//c := cron.New(cron.WithLocation(local), cron.WithSeconds())
-	//eid, err := c.AddFunc(GF.Cron.Spec, test)
-	//if err != nil {
-	//	log.Info("main", zap.Any("AddFunc err", err))
-	//}
-	//log.Info("main", zap.Any("eid", eid))
-	//
-	//c.Start()
-	//defer c.Stop()
+
+	local, _ := time.LoadLocation("Asia/Shanghai")
+	c := cron.New(cron.WithLocation(local), cron.WithSeconds())
+	eid, err := c.AddFunc(GF.Cron.Spec, dealData)
+	if err != nil {
+		log.Info("main", zap.Any("AddFunc err", err))
+	}
+	log.Info("main", zap.Any("eid", eid))
+
+	c.Start()
+	defer c.Stop()
 	//
 	select {}
 }
 
+func dealData() {
+	go dealBidding()
+	go dealProject()
+}
+
 //dealBidding 处理标讯数据
 func dealBidding() {
 	sess := MgoB.GetMgoConn()
@@ -46,6 +51,7 @@ func dealBidding() {
 	var q interface{}
 	var startTime = GF.Cron.Start
 
+	now := time.Now()
 	ctx, _ := context.WithTimeout(context.Background(), 99999*time.Hour)
 	coll := sess.M.C.Database(GF.MongoB.DB).Collection(GF.MongoB.Coll)
 	find := options.Find().SetBatchSize(1000).SetSort(bson.D{bson.E{"comeintime", 1}}).SetProjection(bson.M{"comeintime": 1, "toptype": 1, "subtype": 1, "buyerclass": 1, "title": 1, "detail": 1, "package": 1, "funds": 1})
@@ -56,12 +62,19 @@ func dealBidding() {
 				"$lte": GF.Cron.End,
 			},
 		}
-	} else {
+	} else if startTime != 0 {
 		q = map[string]interface{}{
 			"comeintime": map[string]interface{}{
 				"$gt": startTime,
 			},
 		}
+	} else if startTime == 0 && GF.Cron.End == 0 {
+		//默认 取大于 昨天的数据
+		q = map[string]interface{}{
+			"comeintime": map[string]interface{}{
+				"$gt": time.Date(now.Year(), now.Month(), now.Day()-1, 0, 0, 0, 0, now.Location()).Unix(),
+			},
+		}
 	}
 	cur, err := coll.Find(ctx, q, find)
 	if err != nil {
@@ -134,15 +147,26 @@ func dealProject() {
 	now := time.Now()
 
 	var q interface{}
-	//targetTime := time.Date(now.Year(), now.Month(), now.Day()+GF.Cron.Start, 04, 20, 0, 0, now.Location())
-	targetTime := time.Date(now.Year(), now.Month(), now.Day()+GF.Cron.Start, 0, 0, 0, 0, now.Location())
-	todayTime := time.Date(now.Year(), now.Month(), now.Day()+GF.Cron.End, 0, 0, 0, 0, now.Location())
+	var startTime = GF.Cron.Start
 
-	if GF.Cron.Start != 0 {
+	if startTime != 0 && GF.Cron.End != 0 {
+		q = map[string]interface{}{
+			"pici": map[string]interface{}{
+				"$gt":  GF.Cron.Start,
+				"$lte": GF.Cron.End,
+			},
+		}
+	} else if startTime != 0 {
+		q = map[string]interface{}{
+			"pici": map[string]interface{}{
+				"$gt": startTime,
+			},
+		}
+	} else if startTime == 0 && GF.Cron.End == 0 {
+		//默认 取大于 昨天的数据
 		q = map[string]interface{}{
 			"pici": map[string]interface{}{
-				"$gt":  targetTime.Unix(),
-				"$lte": todayTime.Unix(),
+				"$gt": time.Date(now.Year(), now.Month(), now.Day()-1, 0, 0, 0, 0, now.Location()).Unix(),
 			},
 		}
 	}

website_columns/readme.txt

@@ -1,6 +1,13 @@
 
 >| 网站栏目处理
 
+使用：
+    start end  都不为空，就处理时间段内的数据，需要都是时间戳格式
+    start 不为空，end 为空，就取大于该时间的数据
+    start end 都是空的，就取大于昨天凌晨的数据
+
+
+
 目的： 为了 添加网站栏目 导航，需要给 `bidding` 数据 以及  拟建项目数据表 `projectset_proposed` 打标签
 
 涉及到的标签有：