异步生成数据

JySEPlatform/client/customerRule.go

@@ -23,11 +23,12 @@ import (
 
 type CustomerRule struct {
 	*xweb.Action
-	cuserRuleCreate xweb.Mapper `xweb:"/customerRule/cuser/rule/create"`  //新建规则
-	ruleImport      xweb.Mapper `xweb:"/customerRule/rule/import"`        //导入关键词  导入文件excel
-	productData     xweb.Mapper `xweb:"/customerRule/cuser/produce"`      //生成预览数据
-	demoData        xweb.Mapper `xweb:"/customerRule/rule/preview"`       //预览数据
-	dataTest        xweb.Mapper `xweb:"/customerRule/rule/dataTest/(.*)"` //规则测试
+	cuserRuleCreate xweb.Mapper `xweb:"/customerRule/cuser/rule/create"`    //新建规则
+	ruleImport      xweb.Mapper `xweb:"/customerRule/rule/import"`          //导入关键词  导入文件excel
+	productData     xweb.Mapper `xweb:"/customerRule/cuser/produce"`        //生成预览数据
+	productResult   xweb.Mapper `xweb:"/customerRule/cuser/product/result"` //生成预览数据 结果
+	demoData        xweb.Mapper `xweb:"/customerRule/rule/preview"`         //预览数据
+	dataTest        xweb.Mapper `xweb:"/customerRule/rule/dataTest/(.*)"`   //规则测试
 }
 
 func (c *CustomerRule) CuserRuleCreate() {
@@ -187,7 +188,6 @@ func (c *CustomerRule) ProductData() {
 	defer qu.Catch()
 	if c.Method() == "POST" {
 		log.Println("正在生成预览数据。。。")
-		rep := false
 		id := c.GetString("id")
 		tag, ok := Mgo.FindById("entniche_rule", id, `{}`)
 		if !ok {
@@ -254,8 +254,6 @@ func (c *CustomerRule) ProductData() {
 		// 其他的分隔
 		// 1000<=总个数<4000，将时间（publishtime）按照每3个月进行分割，然后查询到的数据量进行求和。
 		// 4000<=总个数<8000，将时间（publishtime）按照每1个月进行分割，然后查询到的数据量进行求和
-		var err error
-		var count int64
 		// var n int
 		// 小于1000的直接查
 		// if totalCount < 1000 {
@@ -279,7 +277,10 @@ func (c *CustomerRule) ProductData() {
 		// 	listLens := int(redis.LLEN("datag", "jyqyfw_es_query"))
 		// 	if listLens < 2 {
 		// 		redis.RPUSH("datag", "jyqyfw_es_query", 1)
-		err, count = UtilEsFind1(*tag)
+
+		go func(m map[string]interface{}) {
+			UtilEsFind1(m)
+		}(*tag)
 		// 		redis.LPOP("datag", "jyqyfw_es_query")
 		// 		break
 		// 	} else if times2 > 10 {
@@ -292,21 +293,54 @@ func (c *CustomerRule) ProductData() {
 		// 	time.Sleep(2 * time.Second)
 		// }
 
-		if err == nil {
-			rep = true
-			// msg = "数据生成成功"
-			c.ServeJson(map[string]interface{}{
-				"rep":   rep,
-				"count": count,
-			})
-		} else {
-			rep = false
-			msg := "数据生成失败，请稍后再试"
-			c.ServeJson(map[string]interface{}{
-				"rep":   rep,
-				"count": count,
-				"msg":   msg,
-			})
+		c.ServeJson(map[string]interface{}{
+			"rep": true,
+		})
+
+		//if err == nil {
+		//	rep = true
+		//	// msg = "数据生成成功"
+		//	c.ServeJson(map[string]interface{}{
+		//		"rep":   rep,
+		//		"count": count,
+		//	})
+		//} else {
+		//	rep = false
+		//	msg := "数据生成失败，请稍后再试"
+		//	c.ServeJson(map[string]interface{}{
+		//		"rep":   rep,
+		//		"count": count,
+		//		"msg":   msg,
+		//	})
+		//}
+	}
+}
+
+func (c *CustomerRule) ProductResult() {
+	defer qu.Catch()
+	if c.Method() == "POST" {
+		id := c.GetString("id")
+		info, _ := Mgo.FindById("entniche_rule", id, bson.M{"data_result": 1, "i_estotal": 1})
+		if info != nil && len(*info) > 0 {
+			if qu.IntAll((*info)["data_result"]) == 1 {
+				c.ServeJson(map[string]interface{}{
+					"rep":    true,
+					"status": "success",
+					"count":  qu.IntAll((*info)["i_estotal"]),
+				})
+			} else if qu.IntAll((*info)["data_result"]) == -1 {
+				c.ServeJson(map[string]interface{}{
+					"rep":    true,
+					"status": "error",
+					"msg":    "数据生成失败",
+				})
+			} else {
+				c.ServeJson(map[string]interface{}{
+					"rep":    true,
+					"status": "processing",
+					"msg":    "",
+				})
+			}
 		}
 	}
 }

JySEPlatform/config.json

@@ -219,7 +219,7 @@
     "jyMDBXQQIDCQBeSUdBITNf",
     "jyFApXQQIEAw5TTUZOMBpD"
   ],
-  "redis_addrs": "datag=172.20.45.129,export=172.20.45.129:1712,session=172.20.45.129:1713",
+  "redis_addrs": "datag=172.20.45.129:1712,export=172.20.45.129:1712,session=172.20.45.129:1713",
   "jyMysql": {
     "username": "root",
     "password": "=PDT49#80Z!RVv52_z",
@@ -233,7 +233,7 @@
     "size": 15,
     "username": "",
     "password": "",
-    "replSet": "bidding"
+    "replSet": ""
   },
   "file_downloadurl": "http://jy-datafile.oss-cn-beijing.aliyuncs.com/",
   "filter_filehref_appid": [

JySEPlatform/service/customer_service.go

@@ -704,7 +704,7 @@ func (c *Customer) CuserRuleSetup() {
 func (c *Customer) ProductData() {
 	defer qu.Catch()
 	if c.Method() == "POST" {
-		rep := false
+		//rep := false
 		id := c.GetString("id")
 		tag, ok := Mgo.FindById("cuserdepartrule", id, `{}`)
 		if !ok {
@@ -768,20 +768,20 @@ func (c *Customer) ProductData() {
 			}
 
 		}
-		err, count := UtilEsFind1(*tag)
-		var msg string
-		if err == nil {
-			rep = true
-			msg = "数据生成成功"
-		} else {
-			rep = false
-			msg = "数据生成失败"
-		}
-		c.ServeJson(map[string]interface{}{
-			"rep":   rep,
-			"count": count,
-			"msg":   msg,
-		})
+		//err, count := UtilEsFind1(*tag)
+		//var msg string
+		//if err == nil {
+		//	rep = true
+		//	msg = "数据生成成功"
+		//} else {
+		//	rep = false
+		//	msg = "数据生成失败"
+		//}
+		//c.ServeJson(map[string]interface{}{
+		//	"rep":   rep,
+		//	"count": count,
+		//	"msg":   msg,
+		//})
 	}
 }
 

JySEPlatform/util/utiltag.go

@@ -56,6 +56,7 @@ const (
 
 func UtilEsSaveData(sdataid string, datas *[]map[string]interface{}) error {
 	defer qu.Catch()
+	log.Println("保存数据---", sdataid)
 	if !Mgo.Del("tagsdata", bson.M{"s_dataid": sdataid}) {
 		return errors.New("mongo del err")
 	}
@@ -80,9 +81,11 @@ func (m *MySource) Source() (interface{}, error) {
 	return mp["query"], nil
 }
 
+// data_result， 0: 数据生成中，1: 成功，-1: 失败
 // 客户规则
-func UtilEsFind1(tags map[string]interface{}) (error, int64) {
+func UtilEsFind1(tags map[string]interface{}) {
 	defer qu.Catch()
+	Mgo.UpdateById("entniche_rule", tags["_id"], bson.M{"$set": bson.M{"data_result": 0}})
 	ch := make(chan bool, 20)
 	wg := &sync.WaitGroup{}
 	arrsync := &sync.RWMutex{}
@@ -94,7 +97,9 @@ func UtilEsFind1(tags map[string]interface{}) (error, int64) {
 	clearKey := qu.ObjToString(tags["s_globalclearkey"])
 	clearKeyMatch := qu.ObjToString(tags["s_globalclearkeymatch"])
 	if len(esquery) < 1 || len(sdataid) < 1 {
-		return errors.New("s_esquery or s_dataid no found"), 0
+		log.Println("s_esquery or s_dataid no found")
+		Mgo.UpdateById("entniche_rule", tags["_id"], bson.M{"$set": bson.M{"data_result": -1}})
+		return
 	}
 	i_maxnum := qu.Int64All(tags["i_maxnum"])
 	if i_maxnum <= 0 {
@@ -118,7 +123,9 @@ func UtilEsFind1(tags map[string]interface{}) (error, int64) {
 			})
 		}
 	} else {
-		return errors.New("o_rules no found"), 0
+		log.Println("o_rules no found")
+		Mgo.UpdateById("entniche_rule", tags["_id"], bson.M{"$set": bson.M{"data_result": -1}})
+		return
 	}
 	ctx, _ := context.WithTimeout(context.Background(), 1*time.Hour)
 	esCon := elastic.VarEs.(*elastic.EsV7)
@@ -139,10 +146,8 @@ func UtilEsFind1(tags map[string]interface{}) (error, int64) {
 			go func(tmpHit *es.SearchHit) {
 				defer func() {
 					<-ch
-					arrsync.Unlock()
 					wg.Done()
 				}()
-				arrsync.Lock()
 				item := make(map[string]interface{})
 				if json.Unmarshal(tmpHit.Source, &item) == nil {
 					item["appid"] = tags["appid"]
@@ -150,8 +155,6 @@ func UtilEsFind1(tags map[string]interface{}) (error, int64) {
 					info_id := qu.ObjToString(item["id"])
 					item["info_id"] = info_id
 					item["s_dataid"] = sdataid
-					// item["s_jyhref"] = fmt.Sprintf(Url, util.CommonEncodeArticle("content", info_id))
-					//
 					subType := qu.ObjToString(item["subtype"])
 					subTypeStr := "拟建,采购意向,预告,预审,预审结果,论证意见,需求公示"
 					if strings.Contains(subTypeStr, subType) {
@@ -223,8 +226,8 @@ func UtilEsFind1(tags map[string]interface{}) (error, int64) {
 							}
 						}
 					}
-					//匹配公告附件
-					info, _ := MgoBidding.FindOne("bidding", bson.M{"_id": mongodb.StringTOBsonId(info_id)})
+					//匹配公告附件和链接
+					info, _ := MgoBidding.FindOneByField("bidding", bson.M{"_id": mongodb.StringTOBsonId(info_id)}, bson.M{"projectinfo": 1, "competehref": 1})
 					if info != nil {
 						log.Println("匹配附件， id-", item["info_id"])
 						if (*info)["projectinfo"] != nil {
@@ -261,6 +264,13 @@ func UtilEsFind1(tags map[string]interface{}) (error, int64) {
 								}
 							}
 						}
+						if (*info)["competehref"] != nil {
+							if href := qu.ObjToString(item["href"]); strings.Contains(href, "https://www.jianyu360.cn") {
+								if qu.ObjToString(item["competehref"]) != "" && qu.ObjToString(item["competehref"]) != "#" {
+									item["href"] = qu.ObjToString(item["competehref"])
+								}
+							}
+						}
 					} else {
 						log.Println("id", item["info_id"], "bidding为空")
 					}
@@ -269,7 +279,10 @@ func UtilEsFind1(tags map[string]interface{}) (error, int64) {
 					//if projectId != "" {
 					//	item["projectId"] = SE2.EncodeString(projectId)
 					//}
+
+					arrsync.Lock()
 					datas = append(datas, item)
+					arrsync.Unlock()
 				} else {
 					log.Println("id", item["_id"], "信息为空")
 				}
@@ -282,10 +295,11 @@ func UtilEsFind1(tags map[string]interface{}) (error, int64) {
 			"$set": bson.M{
 				"i_estotal": count,
 			}}, false, false)
-		return UtilEsSaveData(sdataid, &datas), count
+		UtilEsSaveData(sdataid, &datas)
+		Mgo.UpdateById("entniche_rule", tags["_id"], bson.M{"$set": bson.M{"data_result": 1}})
 	} else {
 		log.Println(err)
-		return err, 0
+		Mgo.UpdateById("entniche_rule", tags["_id"], bson.M{"$set": bson.M{"data_result": -1}})
 	}
 }
 

JySEPlatform/web/templates/client/cuser_rule_edit.html

@@ -1697,43 +1697,61 @@
         })
     }
 
+    let pollingInterval = null
     //生成数据
     function produceData() {
         com.maskShow("正在生成数据...");
-        var isOk = true
-        setTimeout(function(){
-            $.ajax({
-                url: "/seplatform/customerRule/cuser/produce",
-                type: "post",
-                data: {"id": dataMap.id},
-                async: false,
-                success: function (r) {
-                    if (r.msg){
-                        com.maskHide();
-                        isOk = false
-                        showTip(r.msg, 30000);
-                    }
-                },
-                error: function (r) {
-                    com.maskHide();
+        $.ajax({
+            url: "/seplatform/customerRule/cuser/produce",
+            type: "post",
+            data: {"id": dataMap.id},
+            async: false,
+            success: function (r) {
+                if (r.rep){
+                    // 轮询请求
+                    pollingInterval = setInterval(pollServer, 5000);
+                }else {
+                    showTip(r.msg, 30000);
                 }
-            })
-            if (isOk) {
-                com.maskShow("倒计时3s");
-                setTimeout(function(){
-                    com.maskShow("倒计时2s");
+            },
+            error: function (r) {
+                com.maskHide();
+            }
+        })
+    }
+
+    function pollServer() {
+        $.ajax({
+            url: '/seplatform/customerRule/cuser/product/result',  // 后端接口地址
+            type: 'post',
+            data: {"id": dataMap.id},
+            success: function(r) {
+                if (r.status === 'success') {
+                    com.maskHide();
+                    isOk = true;
+                    clearInterval(pollingInterval);  // 任务完成，停止轮询
+
+                    com.maskShow("倒计时3s");
                     setTimeout(function(){
-                        com.maskShow("倒计时1s");
+                        com.maskShow("倒计时2s");
                         setTimeout(function(){
-                            com.maskShow("倒计时0s");
-                            var href = previewHref+"?type=private" + "&key=" + dataMap.s_dataid;
-                            window.open(href);
-                            com.maskHide();
+                            com.maskShow("倒计时1s");
+                            setTimeout(function(){
+                                com.maskShow("倒计时0s");
+                                exportPreview();
+                                com.maskHide();
+                            },1000)
                         },1000)
                     },1000)
-                },1000)
+
+                } else if (r.status === 'error') {
+                    showTip(r.msg, 30000);
+                }
+            },
+            error: function(r) {
+                com.maskHide();
             }
-        },1000)
+        });
     }
     
     function exportPreview(){